<BR>org.eclipse.cdt.core<BR>

&nbsp;&nbsp;Removed unused methods from IToken.<BR>
&nbsp;&nbsp;Restructured Token implementation to be more memory efficient.<BR>
&nbsp;&nbsp;Tightened up visibility of different members in the implementation.<BR>
&nbsp;&nbsp;Added TokenFactory construct to isolate IToken clients from the particulars of which IToken implementation to instantiate.<BR><BR>
org.eclipse.cdt.core.tests<BR>
&nbsp;&nbsp;Updated ScannerTestCase to correspond with changes to IToken and its implementations.

core/org.eclipse.cdt.core.tests/ChangeLog

@@ -1,3 +1,6 @@
+2004-04-10 John Camelon
+	Updated ScannerTestCase to correspond with changes to IToken and its implementations. 
+
 2004-04-09 Andrew Niefer
 	added parser/org/eclipse/cdt/core/parser/tests/CompleteParseASTTemplateTest.testTemplateMemberTemplateDefinition()
 

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java

@@ -25,7 +25,7 @@ import org.eclipse.cdt.core.parser.ParserFactoryError;
 import org.eclipse.cdt.core.parser.ParserMode;
 import org.eclipse.cdt.core.parser.ScannerException;
 import org.eclipse.cdt.core.parser.ast.IASTInclusion;
-import org.eclipse.cdt.internal.core.parser.token.Token;
+import org.eclipse.cdt.internal.core.parser.token.SimpleToken;
 
 /**
  * @author jcamelon
@@ -734,11 +734,11 @@ public class ScannerTestCase extends BaseScannerTest
 			List expansion= descriptor.getTokenizedExpansion();
 			assertNotNull(parms);
 			assertTrue(expansion.size() == 3);
-			assertTrue(((Token) expansion.get(0)).type == IToken.tIDENTIFIER);
+			assertTrue(((SimpleToken) expansion.get(0)).getType() == IToken.tIDENTIFIER);
-			assertTrue(((Token) expansion.get(0)).getImage().equals("x"));
+			assertTrue(((SimpleToken) expansion.get(0)).getImage().equals("x"));
-			assertTrue(((Token) expansion.get(1)).type == IToken.tPLUS);
+			assertTrue(((SimpleToken) expansion.get(1)).getType() == IToken.tPLUS);
-			assertTrue(((Token) expansion.get(2)).type == IToken.tINTEGER);
+			assertTrue(((SimpleToken) expansion.get(2)).getType() == IToken.tINTEGER);
-			assertTrue(((Token) expansion.get(2)).getImage().equals("1"));
+			assertTrue(((SimpleToken) expansion.get(2)).getImage().equals("1"));
 
 			validateIdentifier("y");
 			validateToken(IToken.tASSIGN);

core/org.eclipse.cdt.core/parser/ChangeLog-parser

@@ -1,3 +1,9 @@
+2004-04-10 John Camelon
+	Removed unused methods from IToken.
+	Restructured Token implementation to be more memory efficient.
+	Tightened up visibility of different members in the implementation.
+	Added TokenFactory construct to isolate Token clients from the particulars of which IToken implementation to instantiate.
+
 2004-04-10 John Camelon
 	Partial fix for https://bugs.eclipse.org/bugs/show_bug.cgi?id=57898
 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java

@@ -16,24 +16,30 @@ package org.eclipse.cdt.core.parser;
  *
  */
 public interface IToken {
-	public abstract String toString();
-	public abstract int getType();
-	public abstract String getImage();
-	public void setImage( String i ); 
-	public abstract int getOffset();
-	public abstract int getLength();
-	public abstract int getEndOffset();
 	
+	// getters
+	public int getType();
+	public String getImage();
+	public int getOffset();
+	public int getLength();
+	public int getEndOffset();
 	// NOTE:if the token spans lines due to escaped newlines then 
 	// the line number returned is the last one 
-	public int getLineNumber();  
+	public int getLineNumber();
-	public abstract int getDelta(IToken other);
+	public IToken getNext();
-	public abstract IToken getNext();
+	
-	public abstract void setNext(IToken t);
+	// setters
-	public abstract void setType(int i);	
+	public void setImage( String i ); 
-	public abstract boolean looksLikeExpression();
+	public void setNext(IToken t);
-	public abstract boolean isPointer();
+	public void setType(int i);	
-	public abstract boolean isOperator();
+
+	// queries
+	public boolean looksLikeExpression();
+	public boolean isPointer();
+	public boolean isOperator();
+	public boolean isKeywordOrOperator();
+	
+	
 	// Token types
 	static public final int tIDENTIFIER = 1;
 
@@ -307,9 +313,4 @@ public interface IToken {
 
 	static public final int tLAST = t_restrict;
 
-	/**
-	 * @return
-	 */
-	public abstract boolean isKeywordOrOperator();
-
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ContextualParser.java

@@ -26,8 +26,8 @@ import org.eclipse.cdt.core.parser.ast.IASTNode;
 import org.eclipse.cdt.core.parser.ast.IASTScope;
 import org.eclipse.cdt.core.parser.ast.IASTCompletionNode.CompletionKind;
 import org.eclipse.cdt.internal.core.parser.token.KeywordSets;
-import org.eclipse.cdt.internal.core.parser.token.Token;
 import org.eclipse.cdt.internal.core.parser.token.TokenDuple;
+import org.eclipse.cdt.internal.core.parser.token.TokenFactory;
 import org.eclipse.cdt.internal.core.parser.token.KeywordSets.Key;
 
 /**
@@ -107,7 +107,7 @@ public class ContextualParser extends CompleteParser {
 		setCompletionKind(kind);
 		setCompletionContext(null);
 		setCompletionFunctionName( );
-		setCompletionToken( new Token( IToken.tIDENTIFIER, prefix ) );
+		setCompletionToken( TokenFactory.createToken( IToken.tIDENTIFIER, prefix ) );
 	}
 
 	/**
@@ -132,7 +132,7 @@ public class ContextualParser extends CompleteParser {
 	}
 
 	protected void setCompletionValues(IASTScope scope, CompletionKind kind, Key key, IASTNode node, String prefix) throws EndOfFileException {
-		setCompletionToken( new Token( IToken.tIDENTIFIER, prefix ) );
+		setCompletionToken( TokenFactory.createToken( IToken.tIDENTIFIER, prefix ) );
 		setCompletionValues(scope, kind, key, node );
 	}
 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java

@@ -40,7 +40,7 @@ import org.eclipse.cdt.core.parser.ast.IASTTypeId;
 import org.eclipse.cdt.core.parser.ast.IASTCompletionNode.CompletionKind;
 import org.eclipse.cdt.core.parser.ast.IASTExpression.Kind;
 import org.eclipse.cdt.internal.core.parser.token.KeywordSets;
-import org.eclipse.cdt.internal.core.parser.token.Token;
+import org.eclipse.cdt.internal.core.parser.token.SimpleToken;
 import org.eclipse.cdt.internal.core.parser.token.TokenDuple;
 import org.eclipse.cdt.internal.core.parser.token.KeywordSets.Key;
 import org.eclipse.cdt.internal.core.parser.util.TraceUtil;
@@ -2674,7 +2674,7 @@ public class ExpressionParser implements IExpressionParser {
 	 *  
 	 */
 	protected void backup(IToken mark) {
-	    currToken = (Token)mark;
+	    currToken = (SimpleToken)mark;
 	    lastToken = null; // this is not entirely right ... 
 	}
 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ObjectMacroDescriptor.java

@@ -14,7 +14,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.eclipse.cdt.core.parser.IMacroDescriptor;
-import org.eclipse.cdt.internal.core.parser.token.Token;
+import org.eclipse.cdt.core.parser.IToken;
 
 /**
  * @author jcamelon
@@ -24,7 +24,7 @@ public class ObjectMacroDescriptor implements IMacroDescriptor {
 	private static final ArrayList EMPTY_LIST = new ArrayList();
 	private final String expansionSignature;
 	private final String name;
-	private final Token token;
+	private final IToken token;
 
 	public ObjectMacroDescriptor( String name, String expansionSignature )
 	{
@@ -33,7 +33,7 @@ public class ObjectMacroDescriptor implements IMacroDescriptor {
 		token = null;
 	}
 	
-	public ObjectMacroDescriptor( String name, Token t, String expansionSignature )
+	public ObjectMacroDescriptor( String name, IToken t, String expansionSignature )
 	{
 		this.name = name;
 		this.token = t;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java

@@ -58,7 +58,8 @@ import org.eclipse.cdt.internal.core.parser.IExpressionParser;
 import org.eclipse.cdt.internal.core.parser.InternalParserUtil;
 import org.eclipse.cdt.internal.core.parser.ast.ASTCompletionNode;
 import org.eclipse.cdt.internal.core.parser.token.KeywordSets;
-import org.eclipse.cdt.internal.core.parser.token.Token;
+import org.eclipse.cdt.internal.core.parser.token.SimpleToken;
+import org.eclipse.cdt.internal.core.parser.token.TokenFactory;
 import org.eclipse.cdt.internal.core.parser.util.TraceUtil;
 
 /**
@@ -487,25 +488,14 @@ public class Scanner implements IScanner {
 			storageBuffer = null; 
 	}
 
-	protected IToken newToken(int t, String i, IScannerContext c) {
+	protected IToken newToken(int t, String i) {
-		setCurrentToken(new Token(t, i, c, scannerData.getContextStack().getCurrentLineNumber()));
+		IToken theToken = TokenFactory.createToken( t, i, scannerData );
+		setCurrentToken(theToken);
 		return currentToken;
 	}
 
 	protected IToken newConstantToken(int t) {
-		setCurrentToken(
+		setCurrentToken( TokenFactory.createToken(t,scannerData));
-				new Token(
-						t, 
-						scannerData.getContextStack().getCurrentContext(),
-						scannerData.getContextStack().getCurrentLineNumber()
-						)
-				);
-		return currentToken;
-	}
-
-
-	protected IToken newToken(int t, String i) {
-		setCurrentToken(new Token(t, i));
 		return currentToken;
 	}
 	
@@ -1017,7 +1007,7 @@ public class Scanner implements IScanner {
 		//If the next token is going to be a string as well, we need to concatenate
 		//it with this token.  This will be recursive for as many strings as need to be concatenated
 		
-		IToken returnToken = newToken( type, buff.toString(), scannerData.getContextStack().getCurrentContext());
+		IToken returnToken = newToken( type, buff.toString());
 			
 		IToken next = null;
 		try{
@@ -1177,8 +1167,7 @@ public class Scanner implements IScanner {
 		
 		return newToken(
 			tokenType,
-			result,
+			result);
-			scannerData.getContextStack().getCurrentContext());
 	}
 	public IToken processPreprocessor() throws ScannerException, EndOfFileException
 	{
@@ -1573,7 +1562,7 @@ public class Scanner implements IScanner {
 		if (tokenTypeObject != null)
 			return newConstantToken(((Integer) tokenTypeObject).intValue());
 		else
-			return newToken(IToken.tIDENTIFIER, ident, scannerData.getContextStack().getCurrentContext());
+			return newToken(IToken.tIDENTIFIER, ident);
 	}
 	
 	/**
@@ -2143,7 +2132,7 @@ public class Scanner implements IScanner {
         	c = getChar(true);
         }
         
-        return newToken( type, buffer.toString(), scannerData.getContextStack().getCurrentContext());                      
+        return newToken( type, buffer.toString());                      
     }
 
 
@@ -2196,7 +2185,7 @@ public class Scanner implements IScanner {
 
                 if (c != NOCHAR ) 
                 {
-                    return newToken( IToken.tSTRING, buff.toString(), scannerData.getContextStack().getCurrentContext());
+                    return newToken( IToken.tSTRING, buff.toString());
     
                 } else {
                 	handleProblem( IProblem.SCANNER_UNBOUNDED_STRING, null, beginOffset, false, true );
@@ -2211,13 +2200,13 @@ public class Scanner implements IScanner {
 	                    return processCharacterLiteral( c, false );
                     case ',' :
                         if (tokenImage.length() > 0) throw endOfMacroToken;
-                        return newToken(IToken.tCOMMA, ",", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$
+                        return newToken(IToken.tCOMMA, ","); //$NON-NLS-1$
                     case '(' :
                         if (tokenImage.length() > 0) throw endOfMacroToken;
-                        return newToken(IToken.tLPAREN, "(", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$
+                        return newToken(IToken.tLPAREN, "("); //$NON-NLS-1$
                     case ')' :
                         if (tokenImage.length() > 0) throw endOfMacroToken;
-                        return newToken(IToken.tRPAREN, ")", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$
+                        return newToken(IToken.tRPAREN, ")"); //$NON-NLS-1$
                     case '/' :
                         if (tokenImage.length() > 0) throw endOfMacroToken;
                         c = getChar();
@@ -2247,7 +2236,7 @@ public class Scanner implements IScanner {
         
         // return completed token
         if (tokenImage.length() > 0) {
-            return newToken(IToken.tIDENTIFIER, tokenImage.toString(), scannerData.getContextStack().getCurrentContext());
+            return newToken(IToken.tIDENTIFIER, tokenImage.toString());
         }
         
         // we're done
@@ -2678,16 +2667,9 @@ public class Scanner implements IScanner {
 	}
 	
 	protected IMacroDescriptor createObjectMacroDescriptor(String key, String value ) {
-		Token t = null;
+		IToken t = null;
 		if( !value.trim().equals( "" ) )  //$NON-NLS-1$
-		{	
+			t = TokenFactory.createToken( IToken.tIDENTIFIER, value, scannerData );
-			t = new Token(
-					IToken.tIDENTIFIER, 
-					value, 
-					scannerData.getContextStack().getCurrentContext(),
-					scannerData.getContextStack().getCurrentLineNumber()
-					);
-		}
 	
 		return new ObjectMacroDescriptor( key,  
 				t, 
@@ -2883,7 +2865,7 @@ public class Scanner implements IScanner {
         
         tokenizer.setThrowExceptionOnBadCharacterRead(false);
         Vector parameterValues = new Vector();
-        Token t = null;
+        SimpleToken t = null;
         StringBuffer buffer = new StringBuffer();
         boolean space = false;
         int nParen = 0;
@@ -2896,12 +2878,12 @@ public class Scanner implements IScanner {
 				}
 				if (c != NOCHAR) tokenizer.ungetChar(c);
 				
-                t = (Token)(forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false));
+                t = (SimpleToken)(forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false));
-                if (t.type == IToken.tLPAREN) {
+                if (t.getType() == IToken.tLPAREN) {
                     nParen++;
-                } else if (t.type == IToken.tRPAREN) {
+                } else if (t.getType() == IToken.tRPAREN) {
                     nParen--;
-                } else if (t.type == IToken.tCOMMA && nParen == 0) {
+                } else if (t.getType() == IToken.tCOMMA && nParen == 0) {
                     parameterValues.add(buffer.toString());
                     buffer = new StringBuffer();
                     space = false;
@@ -2911,7 +2893,7 @@ public class Scanner implements IScanner {
                 if (space)
                     buffer.append( ' ' );
 
-                switch (t.type) {
+                switch (t.getType()) {
                     case IToken.tSTRING :
                     	buffer.append('\"');
                     	buffer.append(t.getImage());
@@ -3002,7 +2984,7 @@ public class Scanner implements IScanner {
                 
                 Vector parameterValues = getMacroParameters(betweenTheBrackets, false);
                 Vector parameterValuesForStringizing = getMacroParameters(betweenTheBrackets, true);
-                Token t = null;
+                SimpleToken t = null;
                 
 				// create a string that represents what needs to be tokenized
 				buffer = new StringBuffer();
@@ -3019,8 +3001,8 @@ public class Scanner implements IScanner {
 				int numberOfTokens = tokens.size();
 
 				for (int i = 0; i < numberOfTokens; ++i) {
-					t = (Token) tokens.get(i);
+					t = (SimpleToken) tokens.get(i);
-					if (t.type == IToken.tIDENTIFIER) {
+					if (t.getType() == IToken.tIDENTIFIER) {
 
 						// is this identifier in the parameterNames
 						// list? 
@@ -3033,10 +3015,10 @@ public class Scanner implements IScanner {
 							buffer.append(
 								(String) parameterValues.elementAt(index) );
 						}
-					} else if (t.type == tPOUND) {
+					} else if (t.getType() == tPOUND) {
 						//next token should be a parameter which needs to be turned into
 						//a string literal
-						t = (Token) tokens.get( ++i );
+						t = (SimpleToken) tokens.get( ++i );
 						int index = parameterNames.indexOf(t.getImage());
 						if( index == -1 ){
 							handleProblem( IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, expansion.getName(), getCurrentOffset(), false, true );
@@ -3067,7 +3049,7 @@ public class Scanner implements IScanner {
 							buffer.append('\"');
 						}
 					} else {
-						switch( t.type )
+						switch( t.getType() )
 						{
 							case IToken.tSTRING:
 								buffer.append('\"');

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/AbstractToken.java

@@ -0,0 +1,173 @@
+/**********************************************************************
+ * Copyright (c) 2002-2004 IBM Canada and others.
+ * All rights reserved.   This program and the accompanying materials
+ * are made available under the terms of the Common Public License v0.5
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors: 
+ * IBM Rational Software - Initial API and implementation
+***********************************************************************/
+package org.eclipse.cdt.internal.core.parser.token;
+
+import org.eclipse.cdt.core.parser.IToken;
+
+
+/**
+ * @author johnc
+ */
+public abstract class AbstractToken  {
+
+	public AbstractToken( int type, int lineNumber )
+	{
+		setType( type );
+		this.lineNumber = lineNumber;
+	}
+
+	public AbstractToken( int type )
+	{
+		setType( type );
+	}
+	
+	public String toString() {
+		return "Token=" + getType() + " \"" + getImage() + " @ line:" + getLineNumber() + " offset=" + getOffset(); 	 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
+	}
+	
+	public abstract String getImage();
+	public abstract int getOffset();
+	public abstract int getLength();
+	
+	public int getType() { return type; }
+
+	public void setType(int i) {
+		type = i;
+	}
+
+	public int getLineNumber() {
+		return lineNumber;
+	}
+
+	public int getEndOffset() { return getOffset() + getLength(); }
+
+	protected int type;
+	protected int lineNumber = 1;
+	protected IToken next = null;
+
+	/* (non-Javadoc)
+	 * @see java.lang.Object#equals(java.lang.Object)
+	 */
+	public boolean equals(Object other) {
+		if( other == null ) return false;
+		if( !( other instanceof IToken ) ) 
+			return false;
+		if( ((IToken)other).getType() != getType() ) 
+			return false;
+		if( !(((IToken)other).getImage().equals( getImage() ))) 
+			return false;
+		return true;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.IToken#isKeyword()
+	 */
+	public boolean isKeywordOrOperator() {
+		switch( getType() )
+		{
+			case IToken.tCHAR:
+			case IToken.tFLOATINGPT:
+			case IToken.tIDENTIFIER:
+			case IToken.tINTEGER:
+			case IToken.tSTRING:
+			case IToken.tLSTRING:
+			case IToken.tLCHAR:
+				return false;
+			default:
+				return true;
+		}
+	}
+
+	public boolean looksLikeExpression()
+	{
+		switch( getType() )
+		{
+			case IToken.tINTEGER:
+			case IToken.t_false:
+			case IToken.t_true:
+			case IToken.tSTRING:
+			case IToken.tLSTRING:
+			case IToken.tFLOATINGPT:
+			case IToken.tCHAR:
+			case IToken.tAMPER:
+			case IToken.tDOT:
+			case IToken.tLPAREN:
+			case IToken.tMINUS:
+			case IToken.tSTAR: 
+			case IToken.tPLUS: 
+			case IToken.tNOT:
+			case IToken.tCOMPL:
+				return true;
+			default:
+				break;
+		}	
+		return false;
+	}
+	
+	public boolean isOperator()
+	{
+		switch( getType() )
+		{
+			case IToken.t_new:
+			case IToken.t_delete:
+			case IToken.tPLUS:
+			case IToken.tMINUS:
+			case IToken.tSTAR:
+			case IToken.tDIV:
+			case IToken.tXOR:
+			case IToken.tMOD:
+			case IToken.tAMPER:
+			case IToken.tBITOR:
+			case IToken.tCOMPL:
+			case IToken.tNOT:
+			case IToken.tASSIGN:
+			case IToken.tLT:
+			case IToken.tGT:
+			case IToken.tPLUSASSIGN:
+			case IToken.tMINUSASSIGN:
+			case IToken.tSTARASSIGN:
+			case IToken.tDIVASSIGN:
+			case IToken.tMODASSIGN:
+			case IToken.tBITORASSIGN:
+			case IToken.tAMPERASSIGN:
+			case IToken.tXORASSIGN:
+			case IToken.tSHIFTL:
+			case IToken.tSHIFTR:
+			case IToken.tSHIFTLASSIGN:
+			case IToken.tSHIFTRASSIGN:
+			case IToken.tEQUAL:
+			case IToken.tNOTEQUAL:
+			case IToken.tLTEQUAL:
+			case IToken.tGTEQUAL:
+			case IToken.tAND:
+			case IToken.tOR:
+			case IToken.tINCR:
+			case IToken.tDECR:
+			case IToken.tCOMMA:
+			case IToken.tARROW:
+			case IToken.tARROWSTAR:
+				return true;
+			default:
+				return false;
+		}
+	}
+	
+	public boolean isPointer()
+	{
+		return (getType() == IToken.tAMPER || getType() == IToken.tSTAR);
+	}
+
+
+
+	public final IToken getNext() { return next; }
+	public void setNext(IToken t) { next = t; }
+	
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedExpansionToken.java

@@ -0,0 +1,48 @@
+/**********************************************************************
+ * Copyright (c) 2002-2004 IBM Canada and others.
+ * All rights reserved.   This program and the accompanying materials
+ * are made available under the terms of the Common Public License v0.5
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors: 
+ * IBM Rational Software - Initial API and implementation
+***********************************************************************/
+package org.eclipse.cdt.internal.core.parser.token;
+
+import org.eclipse.cdt.core.parser.IToken;
+import org.eclipse.cdt.internal.core.parser.scanner.ContextStack;
+import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
+
+/**
+ * @author johnc
+ */
+public class ImagedExpansionToken extends ImagedToken implements IToken {
+
+	/**
+	 * @param t
+	 * @param contextStack
+	 * @param i
+	 */
+	public ImagedExpansionToken(int t, ContextStack contextStack, String i) {
+		super(t, contextStack, i);
+	}
+	
+	protected int length;
+	
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.internal.core.parser.token.SimpleToken#setOffsetAndLength(org.eclipse.cdt.internal.core.parser.scanner.IScannerContext)
+	 */
+	protected void setOffsetAndLength(IScannerContext context) {
+		offset = context.getMacroOffset();
+		length = context.getMacroLength();
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getLength()
+	 */
+	public final int getLength() {
+		return length;
+	}
+
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedToken.java

@@ -0,0 +1,78 @@
+/**********************************************************************
+ * Copyright (c) 2002-2004 IBM Canada and others.
+ * All rights reserved.   This program and the accompanying materials
+ * are made available under the terms of the Common Public License v0.5
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors: 
+ * IBM Rational Software - Initial API and implementation
+***********************************************************************/
+package org.eclipse.cdt.internal.core.parser.token;
+
+import org.eclipse.cdt.internal.core.parser.scanner.ContextStack;
+import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
+
+/**
+ * @author johnc
+ */
+public class ImagedToken extends SimpleToken {
+
+	protected String image = null;
+	
+	/**
+	 * @param t
+	 * @param contextStack
+	 * @param i
+	 */
+	public ImagedToken(int t, ContextStack contextStack, String i) {
+		super(t, contextStack);
+		setImage( i );
+		setOffsetAndLength(contextStack.getCurrentContext());
+	}
+	
+	public ImagedToken( int t, String i) {
+		super( t );
+		setImage(i);
+	}
+	
+	
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getImage()
+	 */
+	public final String getImage() {
+		return image;
+	}
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.IToken#setImage(java.lang.String)
+	 */
+	public void setImage(String i) {
+		image = i;
+	}
+	
+	/**
+	 * @param context
+	 */
+	protected void setOffsetAndLength(IScannerContext context) {
+		if( getImage() == null ) return;
+		offset = context.getOffset() - getImage().length() - context.undoStackSize();		
+		if( getType() == tSTRING || getType() == tCHAR )
+			offset--;
+		else if( getType() == tLSTRING || getType() == tLCHAR )
+			offset -= 2;
+	}
+	
+	public int getLength() {
+		switch( getType() )
+		{
+			case tSTRING:
+			case tCHAR:
+				return getImage().length() + 2;  // 'c' is 3 characters, not 1
+			case tLSTRING:
+			case tLCHAR:
+				return getImage().length() + 3;  // L"X" if 4 characters, not 1
+			default:
+				return getImage().length();
+		}
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleExpansionToken.java

@@ -0,0 +1,48 @@
+/**********************************************************************
+ * Copyright (c) 2002-2004 IBM Canada and others.
+ * All rights reserved.   This program and the accompanying materials
+ * are made available under the terms of the Common Public License v0.5
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors: 
+ * IBM Rational Software - Initial API and implementation
+***********************************************************************/
+package org.eclipse.cdt.internal.core.parser.token;
+
+import org.eclipse.cdt.core.parser.IToken;
+import org.eclipse.cdt.internal.core.parser.scanner.ContextStack;
+import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
+
+/**
+ * @author johnc
+ */
+public class SimpleExpansionToken extends SimpleToken implements IToken {
+
+	/**
+	 * @param tokenType
+	 * @param stack
+	 */
+	public SimpleExpansionToken(int tokenType, ContextStack stack) {
+		super( tokenType, stack );
+	}
+	
+	protected int length;
+	
+	
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.internal.core.parser.token.SimpleToken#setOffsetAndLength(org.eclipse.cdt.internal.core.parser.scanner.IScannerContext)
+	 */
+	protected void setOffsetAndLength(IScannerContext context) {
+		offset = context.getMacroOffset();
+		length = context.getMacroLength();
+	}
+	
+	
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getLength()
+	 */
+	public final int getLength() {
+		return length;
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleToken.java

@@ -10,71 +10,45 @@
  ******************************************************************************/
 package org.eclipse.cdt.internal.core.parser.token;
 
-import org.eclipse.cdt.core.parser.Keywords;
 import org.eclipse.cdt.core.parser.IScanner;
 import org.eclipse.cdt.core.parser.IToken;
-import org.eclipse.cdt.internal.core.parser.scanner.*;
+import org.eclipse.cdt.core.parser.Keywords;
+import org.eclipse.cdt.internal.core.parser.scanner.ContextStack;
+import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
 
-public class Token implements IToken {
+public class SimpleToken extends AbstractToken implements IToken {
-
+	
-	public Token(int t, String i, IScannerContext context, int lineNumber ) {
+	public SimpleToken(int t, ContextStack contextStack ) {
-		setType(t);
+		super(t,contextStack.getCurrentLineNumber());
-		setImage(i);
+		setOffsetAndLength(contextStack.getCurrentContext());
-		filename = context.getFilename();
-		offset = context.getOffset() - getImage().length() - context.undoStackSize();
-		this.lineNumber = lineNumber;
-        macroOffset = context.getMacroOffset();
-        macroLength = context.getMacroLength();
-		
-		if( type == tLSTRING || type == tSTRING || type == tCHAR ){
-			offset--;
-		}
-	}
-	public Token(int t, IScannerContext context, int lineNumber ) {
-		setType(t);
-		setImage(null);
-		filename = context.getFilename();
-		offset = context.getOffset() - getImage().length() - context.undoStackSize();
-		this.lineNumber = lineNumber;
-        macroOffset = context.getMacroOffset();
-        macroLength = context.getMacroLength();
-		
-		if( type == tLSTRING || type == tSTRING || type == tCHAR ){
-			offset--;
-		}
 	}
 	
-	public Token(int t) {
+	public SimpleToken( int t )
-		setType(t);
-		setImage(null);
-	}
-	public Token(int t, String i) {
-		setType(t);
-		setImage(i);
-	}
-	
-	public String toString()
 	{
-		return "Token=" + type + " \"" + getImage() + "\" " + filename + ":" + lineNumber + " offset=" + offset; 	 //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+		super( t );
 	}
 
-	public int type;
+	protected int offset;
-	public int getType() { return type; }
+
+    // All the tokens generated by the macro expansion 
+    // will have dimensions (offset and length) equal to the expanding symbol.
+	public int getOffset() { 
+		return offset; 
+	}
 	
-	protected String image;
+	public int getLength() {
+		return getImage().length();
+	}
+	
+	/**
+	 * @param context
+	 */
+	protected void setOffsetAndLength(IScannerContext context) {
+		offset = context.getOffset() - getImage().length() - context.undoStackSize();		
+	}
 	
 	public String getImage() { 
 		switch ( getType() ) {
-			
-				case IToken.tIDENTIFIER :
-				case IToken.tINTEGER :
-				case IToken.tFLOATINGPT :
-				case IToken.tSTRING :			
-				case IToken.tLSTRING :
-				case IToken.tCHAR :			
-				case IToken.tLCHAR :
-					return image;
-
 				case IToken.tCOLONCOLON :
 					return "::" ; //$NON-NLS-1$
 				case IToken.tCOLON :
@@ -336,164 +310,17 @@ public class Token implements IToken {
 				
 				default :
 					// we should never get here!
-					return image; 
+					// assert false : getType();
+					return ""; //$NON-NLS-1$ 
 		}			
 	}
-
+   
-	public  String filename;
-    
-	protected int offset;
-    protected int macroOffset = -1;
-    protected int macroLength = -1;
-    // All the tokens generated by the macro expansion 
-    // will have dimensions (offset and length) equal to the expanding symbol.
-	public int getOffset() { return (macroOffset < 0) ? offset : macroOffset; }
-
-	public int getLength() { return (macroLength < 0) ? getImage().length() : macroLength; }
-
-	public int getEndOffset() { return getOffset() + getLength(); }
 	
-	
-	public int getDelta( IToken other )
-	{
-		return other.getOffset() + other.getLength() - getOffset();
-	}
-	
-	private IToken next;
-
-	private int lineNumber;
-	public IToken getNext() { return next; }
-	public void setNext(IToken t) { next = t; }
-	
-	public boolean looksLikeExpression()
-	{
-		switch( getType() )
-		{
-			case tINTEGER:
-			case t_false:
-			case t_true:
-			case tSTRING:
-			case tLSTRING:
-			case tFLOATINGPT:
-			case tCHAR:
-			case tAMPER:
-			case tDOT:
-			case tLPAREN:
-			case tMINUS:
-			case tSTAR: 
-			case tPLUS: 
-			case tNOT:
-			case tCOMPL:
-				return true;
-			default:
-				break;
-		}
-	
-		
-		return false;
-	}
-	
-	public boolean isPointer()
-	{
-		return (getType() == tAMPER || getType() == tSTAR);
-	}
-	
-	public boolean isOperator()
-	{
-		switch( getType() )
-		{
-			case IToken.t_new:
-			case IToken.t_delete:
-			case IToken.tPLUS:
-			case IToken.tMINUS:
-			case IToken.tSTAR:
-			case IToken.tDIV:
-			case IToken.tXOR:
-			case IToken.tMOD:
-			case IToken.tAMPER:
-			case IToken.tBITOR:
-			case IToken.tCOMPL:
-			case IToken.tNOT:
-			case IToken.tASSIGN:
-			case IToken.tLT:
-			case IToken.tGT:
-			case IToken.tPLUSASSIGN:
-			case IToken.tMINUSASSIGN:
-			case IToken.tSTARASSIGN:
-			case IToken.tDIVASSIGN:
-			case IToken.tMODASSIGN:
-			case IToken.tBITORASSIGN:
-			case IToken.tAMPERASSIGN:
-			case IToken.tXORASSIGN:
-			case IToken.tSHIFTL:
-			case IToken.tSHIFTR:
-			case IToken.tSHIFTLASSIGN:
-			case IToken.tSHIFTRASSIGN:
-			case IToken.tEQUAL:
-			case IToken.tNOTEQUAL:
-			case IToken.tLTEQUAL:
-			case IToken.tGTEQUAL:
-			case IToken.tAND:
-			case IToken.tOR:
-			case IToken.tINCR:
-			case IToken.tDECR:
-			case IToken.tCOMMA:
-			case IToken.tARROW:
-			case IToken.tARROWSTAR:
-				return true;
-			default:
-				return false;
-		}
-	}
-
 	/* (non-Javadoc)
 	 * @see org.eclipse.cdt.core.parser.IToken#setImage()
 	 */
 	public void setImage( String i ) {
-		image = i; 
+		// do nothing
-	}
-	
-
-	/* (non-Javadoc)
-	 * @see java.lang.Object#equals(java.lang.Object)
-	 */
-	public boolean equals(Object other) {
-		if( other == null ) return false;
-		if( !( other instanceof IToken ) ) 
-			return false;
-		if( !(((IToken)other).getImage().equals( getImage() ))) 
-			return false;
-		if( ((IToken)other).getType() != type ) 
-			return false;
-		return true;
-	}
-
-	/* (non-Javadoc)
-	 * @see org.eclipse.cdt.core.parser.IToken#setType(int)
-	 */
-	public void setType(int i) {
-		type = i;
-	}
-
-	/* (non-Javadoc)
-	 * @see org.eclipse.cdt.core.parser.IToken#getLineNumber()
-	 */
-	public int getLineNumber() {
-		return lineNumber;
-	}
-
-	/* (non-Javadoc)
-	 * @see org.eclipse.cdt.core.parser.IToken#isKeyword()
-	 */
-	public boolean isKeywordOrOperator() {
-		if( type == IToken.tCHAR ) return false;
-		if( type == IToken.tFLOATINGPT ) return false;
-		if( type == IToken.tIDENTIFIER ) return false;
-		if( type == IToken.tINTEGER ) return false;
-		if( type == IToken.tSTRING ) return false;
-		if( type == IToken.tLSTRING ) return false;
-		if( type == IToken.tLCHAR ) return false;
-		return true;
 	}
 
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java

@@ -0,0 +1,47 @@
+/**********************************************************************
+ * Copyright (c) 2002-2004 IBM Canada and others.
+ * All rights reserved.   This program and the accompanying materials
+ * are made available under the terms of the Common Public License v0.5
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors: 
+ * IBM Rational Software - Initial API and implementation
+***********************************************************************/
+package org.eclipse.cdt.internal.core.parser.token;
+
+import org.eclipse.cdt.core.parser.IToken;
+import org.eclipse.cdt.internal.core.parser.scanner.IScannerData;
+
+/**
+ * @author johnc
+ */
+public class TokenFactory {
+	
+	public static IToken createToken( int tokenType, IScannerData scannerData )
+	{
+		if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 )
+			return new SimpleExpansionToken( tokenType, scannerData.getContextStack() );
+		
+		return new SimpleToken(	tokenType, scannerData.getContextStack() );
+	}
+
+	/**
+	 * @param type
+	 * @param image
+	 * @param scannerData
+	 * @return
+	 */
+	public static IToken createToken(int type, String image, IScannerData scannerData) {
+		if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 )
+			return new ImagedExpansionToken( type, scannerData.getContextStack(), image );
+
+		return new ImagedToken(type, scannerData.getContextStack(), image );
+	}
+	
+	
+	public static IToken createToken( int type, String image )
+	{
+		return new ImagedToken( type, image);
+	}
+}