diff --git a/core/org.eclipse.cdt.core.tests/ChangeLog b/core/org.eclipse.cdt.core.tests/ChangeLog index 153dffda301..3f739b6eb8d 100644 --- a/core/org.eclipse.cdt.core.tests/ChangeLog +++ b/core/org.eclipse.cdt.core.tests/ChangeLog @@ -1,3 +1,6 @@ +2004-04-10 John Camelon + Updated ScannerTestCase to correspond with changes to IToken and its implementations. + 2004-04-09 Andrew Niefer added parser/org/eclipse/cdt/core/parser/tests/CompleteParseASTTemplateTest.testTemplateMemberTemplateDefinition() diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java index 91628dda42d..a72c6e6cc72 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java @@ -25,7 +25,7 @@ import org.eclipse.cdt.core.parser.ParserFactoryError; import org.eclipse.cdt.core.parser.ParserMode; import org.eclipse.cdt.core.parser.ScannerException; import org.eclipse.cdt.core.parser.ast.IASTInclusion; -import org.eclipse.cdt.internal.core.parser.token.Token; +import org.eclipse.cdt.internal.core.parser.token.SimpleToken; /** * @author jcamelon @@ -734,11 +734,11 @@ public class ScannerTestCase extends BaseScannerTest List expansion= descriptor.getTokenizedExpansion(); assertNotNull(parms); assertTrue(expansion.size() == 3); - assertTrue(((Token) expansion.get(0)).type == IToken.tIDENTIFIER); - assertTrue(((Token) expansion.get(0)).getImage().equals("x")); - assertTrue(((Token) expansion.get(1)).type == IToken.tPLUS); - assertTrue(((Token) expansion.get(2)).type == IToken.tINTEGER); - assertTrue(((Token) expansion.get(2)).getImage().equals("1")); + assertTrue(((SimpleToken) expansion.get(0)).getType() == IToken.tIDENTIFIER); + assertTrue(((SimpleToken) expansion.get(0)).getImage().equals("x")); + assertTrue(((SimpleToken) expansion.get(1)).getType() == IToken.tPLUS); + assertTrue(((SimpleToken) expansion.get(2)).getType() == IToken.tINTEGER); + assertTrue(((SimpleToken) expansion.get(2)).getImage().equals("1")); validateIdentifier("y"); validateToken(IToken.tASSIGN); diff --git a/core/org.eclipse.cdt.core/parser/ChangeLog-parser b/core/org.eclipse.cdt.core/parser/ChangeLog-parser index 590a14f43b4..2be9a6a2b85 100644 --- a/core/org.eclipse.cdt.core/parser/ChangeLog-parser +++ b/core/org.eclipse.cdt.core/parser/ChangeLog-parser @@ -1,3 +1,9 @@ +2004-04-10 John Camelon + Removed unused methods from IToken. + Restructured Token implementation to be more memory efficient. + Tightened up visibility of different members in the implementation. + Added TokenFactory construct to isolate Token clients from the particulars of which IToken implementation to instantiate. + 2004-04-10 John Camelon Partial fix for https://bugs.eclipse.org/bugs/show_bug.cgi?id=57898 diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java index 381eed9e945..6b9a37d59ff 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java @@ -16,24 +16,30 @@ package org.eclipse.cdt.core.parser; * */ public interface IToken { - public abstract String toString(); - public abstract int getType(); - public abstract String getImage(); - public void setImage( String i ); - public abstract int getOffset(); - public abstract int getLength(); - public abstract int getEndOffset(); + // getters + public int getType(); + public String getImage(); + public int getOffset(); + public int getLength(); + public int getEndOffset(); // NOTE:if the token spans lines due to escaped newlines then // the line number returned is the last one - public int getLineNumber(); - public abstract int getDelta(IToken other); - public abstract IToken getNext(); - public abstract void setNext(IToken t); - public abstract void setType(int i); - public abstract boolean looksLikeExpression(); - public abstract boolean isPointer(); - public abstract boolean isOperator(); + public int getLineNumber(); + public IToken getNext(); + + // setters + public void setImage( String i ); + public void setNext(IToken t); + public void setType(int i); + + // queries + public boolean looksLikeExpression(); + public boolean isPointer(); + public boolean isOperator(); + public boolean isKeywordOrOperator(); + + // Token types static public final int tIDENTIFIER = 1; @@ -307,9 +313,4 @@ public interface IToken { static public final int tLAST = t_restrict; - /** - * @return - */ - public abstract boolean isKeywordOrOperator(); - } \ No newline at end of file diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ContextualParser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ContextualParser.java index 7c958797d41..3926a563551 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ContextualParser.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ContextualParser.java @@ -26,8 +26,8 @@ import org.eclipse.cdt.core.parser.ast.IASTNode; import org.eclipse.cdt.core.parser.ast.IASTScope; import org.eclipse.cdt.core.parser.ast.IASTCompletionNode.CompletionKind; import org.eclipse.cdt.internal.core.parser.token.KeywordSets; -import org.eclipse.cdt.internal.core.parser.token.Token; import org.eclipse.cdt.internal.core.parser.token.TokenDuple; +import org.eclipse.cdt.internal.core.parser.token.TokenFactory; import org.eclipse.cdt.internal.core.parser.token.KeywordSets.Key; /** @@ -107,7 +107,7 @@ public class ContextualParser extends CompleteParser { setCompletionKind(kind); setCompletionContext(null); setCompletionFunctionName( ); - setCompletionToken( new Token( IToken.tIDENTIFIER, prefix ) ); + setCompletionToken( TokenFactory.createToken( IToken.tIDENTIFIER, prefix ) ); } /** @@ -132,7 +132,7 @@ public class ContextualParser extends CompleteParser { } protected void setCompletionValues(IASTScope scope, CompletionKind kind, Key key, IASTNode node, String prefix) throws EndOfFileException { - setCompletionToken( new Token( IToken.tIDENTIFIER, prefix ) ); + setCompletionToken( TokenFactory.createToken( IToken.tIDENTIFIER, prefix ) ); setCompletionValues(scope, kind, key, node ); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java index 35fc817b714..ebe1fe9fc4f 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java @@ -40,7 +40,7 @@ import org.eclipse.cdt.core.parser.ast.IASTTypeId; import org.eclipse.cdt.core.parser.ast.IASTCompletionNode.CompletionKind; import org.eclipse.cdt.core.parser.ast.IASTExpression.Kind; import org.eclipse.cdt.internal.core.parser.token.KeywordSets; -import org.eclipse.cdt.internal.core.parser.token.Token; +import org.eclipse.cdt.internal.core.parser.token.SimpleToken; import org.eclipse.cdt.internal.core.parser.token.TokenDuple; import org.eclipse.cdt.internal.core.parser.token.KeywordSets.Key; import org.eclipse.cdt.internal.core.parser.util.TraceUtil; @@ -2674,7 +2674,7 @@ public class ExpressionParser implements IExpressionParser { * */ protected void backup(IToken mark) { - currToken = (Token)mark; + currToken = (SimpleToken)mark; lastToken = null; // this is not entirely right ... } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ObjectMacroDescriptor.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ObjectMacroDescriptor.java index 19e99eb9bbf..61225eb3711 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ObjectMacroDescriptor.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ObjectMacroDescriptor.java @@ -14,7 +14,7 @@ import java.util.ArrayList; import java.util.List; import org.eclipse.cdt.core.parser.IMacroDescriptor; -import org.eclipse.cdt.internal.core.parser.token.Token; +import org.eclipse.cdt.core.parser.IToken; /** * @author jcamelon @@ -24,7 +24,7 @@ public class ObjectMacroDescriptor implements IMacroDescriptor { private static final ArrayList EMPTY_LIST = new ArrayList(); private final String expansionSignature; private final String name; - private final Token token; + private final IToken token; public ObjectMacroDescriptor( String name, String expansionSignature ) { @@ -33,7 +33,7 @@ public class ObjectMacroDescriptor implements IMacroDescriptor { token = null; } - public ObjectMacroDescriptor( String name, Token t, String expansionSignature ) + public ObjectMacroDescriptor( String name, IToken t, String expansionSignature ) { this.name = name; this.token = t; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java index c63d769a1dd..d185a54e580 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java @@ -58,7 +58,8 @@ import org.eclipse.cdt.internal.core.parser.IExpressionParser; import org.eclipse.cdt.internal.core.parser.InternalParserUtil; import org.eclipse.cdt.internal.core.parser.ast.ASTCompletionNode; import org.eclipse.cdt.internal.core.parser.token.KeywordSets; -import org.eclipse.cdt.internal.core.parser.token.Token; +import org.eclipse.cdt.internal.core.parser.token.SimpleToken; +import org.eclipse.cdt.internal.core.parser.token.TokenFactory; import org.eclipse.cdt.internal.core.parser.util.TraceUtil; /** @@ -487,25 +488,14 @@ public class Scanner implements IScanner { storageBuffer = null; } - protected IToken newToken(int t, String i, IScannerContext c) { - setCurrentToken(new Token(t, i, c, scannerData.getContextStack().getCurrentLineNumber())); + protected IToken newToken(int t, String i) { + IToken theToken = TokenFactory.createToken( t, i, scannerData ); + setCurrentToken(theToken); return currentToken; } protected IToken newConstantToken(int t) { - setCurrentToken( - new Token( - t, - scannerData.getContextStack().getCurrentContext(), - scannerData.getContextStack().getCurrentLineNumber() - ) - ); - return currentToken; - } - - - protected IToken newToken(int t, String i) { - setCurrentToken(new Token(t, i)); + setCurrentToken( TokenFactory.createToken(t,scannerData)); return currentToken; } @@ -1017,7 +1007,7 @@ public class Scanner implements IScanner { //If the next token is going to be a string as well, we need to concatenate //it with this token. This will be recursive for as many strings as need to be concatenated - IToken returnToken = newToken( type, buff.toString(), scannerData.getContextStack().getCurrentContext()); + IToken returnToken = newToken( type, buff.toString()); IToken next = null; try{ @@ -1177,8 +1167,7 @@ public class Scanner implements IScanner { return newToken( tokenType, - result, - scannerData.getContextStack().getCurrentContext()); + result); } public IToken processPreprocessor() throws ScannerException, EndOfFileException { @@ -1573,7 +1562,7 @@ public class Scanner implements IScanner { if (tokenTypeObject != null) return newConstantToken(((Integer) tokenTypeObject).intValue()); else - return newToken(IToken.tIDENTIFIER, ident, scannerData.getContextStack().getCurrentContext()); + return newToken(IToken.tIDENTIFIER, ident); } /** @@ -2143,7 +2132,7 @@ public class Scanner implements IScanner { c = getChar(true); } - return newToken( type, buffer.toString(), scannerData.getContextStack().getCurrentContext()); + return newToken( type, buffer.toString()); } @@ -2196,7 +2185,7 @@ public class Scanner implements IScanner { if (c != NOCHAR ) { - return newToken( IToken.tSTRING, buff.toString(), scannerData.getContextStack().getCurrentContext()); + return newToken( IToken.tSTRING, buff.toString()); } else { handleProblem( IProblem.SCANNER_UNBOUNDED_STRING, null, beginOffset, false, true ); @@ -2211,13 +2200,13 @@ public class Scanner implements IScanner { return processCharacterLiteral( c, false ); case ',' : if (tokenImage.length() > 0) throw endOfMacroToken; - return newToken(IToken.tCOMMA, ",", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$ + return newToken(IToken.tCOMMA, ","); //$NON-NLS-1$ case '(' : if (tokenImage.length() > 0) throw endOfMacroToken; - return newToken(IToken.tLPAREN, "(", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$ + return newToken(IToken.tLPAREN, "("); //$NON-NLS-1$ case ')' : if (tokenImage.length() > 0) throw endOfMacroToken; - return newToken(IToken.tRPAREN, ")", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$ + return newToken(IToken.tRPAREN, ")"); //$NON-NLS-1$ case '/' : if (tokenImage.length() > 0) throw endOfMacroToken; c = getChar(); @@ -2247,7 +2236,7 @@ public class Scanner implements IScanner { // return completed token if (tokenImage.length() > 0) { - return newToken(IToken.tIDENTIFIER, tokenImage.toString(), scannerData.getContextStack().getCurrentContext()); + return newToken(IToken.tIDENTIFIER, tokenImage.toString()); } // we're done @@ -2678,16 +2667,9 @@ public class Scanner implements IScanner { } protected IMacroDescriptor createObjectMacroDescriptor(String key, String value ) { - Token t = null; + IToken t = null; if( !value.trim().equals( "" ) ) //$NON-NLS-1$ - { - t = new Token( - IToken.tIDENTIFIER, - value, - scannerData.getContextStack().getCurrentContext(), - scannerData.getContextStack().getCurrentLineNumber() - ); - } + t = TokenFactory.createToken( IToken.tIDENTIFIER, value, scannerData ); return new ObjectMacroDescriptor( key, t, @@ -2883,7 +2865,7 @@ public class Scanner implements IScanner { tokenizer.setThrowExceptionOnBadCharacterRead(false); Vector parameterValues = new Vector(); - Token t = null; + SimpleToken t = null; StringBuffer buffer = new StringBuffer(); boolean space = false; int nParen = 0; @@ -2896,12 +2878,12 @@ public class Scanner implements IScanner { } if (c != NOCHAR) tokenizer.ungetChar(c); - t = (Token)(forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false)); - if (t.type == IToken.tLPAREN) { + t = (SimpleToken)(forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false)); + if (t.getType() == IToken.tLPAREN) { nParen++; - } else if (t.type == IToken.tRPAREN) { + } else if (t.getType() == IToken.tRPAREN) { nParen--; - } else if (t.type == IToken.tCOMMA && nParen == 0) { + } else if (t.getType() == IToken.tCOMMA && nParen == 0) { parameterValues.add(buffer.toString()); buffer = new StringBuffer(); space = false; @@ -2911,7 +2893,7 @@ public class Scanner implements IScanner { if (space) buffer.append( ' ' ); - switch (t.type) { + switch (t.getType()) { case IToken.tSTRING : buffer.append('\"'); buffer.append(t.getImage()); @@ -3002,7 +2984,7 @@ public class Scanner implements IScanner { Vector parameterValues = getMacroParameters(betweenTheBrackets, false); Vector parameterValuesForStringizing = getMacroParameters(betweenTheBrackets, true); - Token t = null; + SimpleToken t = null; // create a string that represents what needs to be tokenized buffer = new StringBuffer(); @@ -3019,8 +3001,8 @@ public class Scanner implements IScanner { int numberOfTokens = tokens.size(); for (int i = 0; i < numberOfTokens; ++i) { - t = (Token) tokens.get(i); - if (t.type == IToken.tIDENTIFIER) { + t = (SimpleToken) tokens.get(i); + if (t.getType() == IToken.tIDENTIFIER) { // is this identifier in the parameterNames // list? @@ -3033,10 +3015,10 @@ public class Scanner implements IScanner { buffer.append( (String) parameterValues.elementAt(index) ); } - } else if (t.type == tPOUND) { + } else if (t.getType() == tPOUND) { //next token should be a parameter which needs to be turned into //a string literal - t = (Token) tokens.get( ++i ); + t = (SimpleToken) tokens.get( ++i ); int index = parameterNames.indexOf(t.getImage()); if( index == -1 ){ handleProblem( IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, expansion.getName(), getCurrentOffset(), false, true ); @@ -3067,7 +3049,7 @@ public class Scanner implements IScanner { buffer.append('\"'); } } else { - switch( t.type ) + switch( t.getType() ) { case IToken.tSTRING: buffer.append('\"'); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/AbstractToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/AbstractToken.java new file mode 100644 index 00000000000..1b4e5829eeb --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/AbstractToken.java @@ -0,0 +1,173 @@ +/********************************************************************** + * Copyright (c) 2002-2004 IBM Canada and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Rational Software - Initial API and implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.token; + +import org.eclipse.cdt.core.parser.IToken; + + +/** + * @author johnc + */ +public abstract class AbstractToken { + + public AbstractToken( int type, int lineNumber ) + { + setType( type ); + this.lineNumber = lineNumber; + } + + public AbstractToken( int type ) + { + setType( type ); + } + + public String toString() { + return "Token=" + getType() + " \"" + getImage() + " @ line:" + getLineNumber() + " offset=" + getOffset(); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ + } + + public abstract String getImage(); + public abstract int getOffset(); + public abstract int getLength(); + + public int getType() { return type; } + + public void setType(int i) { + type = i; + } + + public int getLineNumber() { + return lineNumber; + } + + public int getEndOffset() { return getOffset() + getLength(); } + + protected int type; + protected int lineNumber = 1; + protected IToken next = null; + + /* (non-Javadoc) + * @see java.lang.Object#equals(java.lang.Object) + */ + public boolean equals(Object other) { + if( other == null ) return false; + if( !( other instanceof IToken ) ) + return false; + if( ((IToken)other).getType() != getType() ) + return false; + if( !(((IToken)other).getImage().equals( getImage() ))) + return false; + return true; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#isKeyword() + */ + public boolean isKeywordOrOperator() { + switch( getType() ) + { + case IToken.tCHAR: + case IToken.tFLOATINGPT: + case IToken.tIDENTIFIER: + case IToken.tINTEGER: + case IToken.tSTRING: + case IToken.tLSTRING: + case IToken.tLCHAR: + return false; + default: + return true; + } + } + + public boolean looksLikeExpression() + { + switch( getType() ) + { + case IToken.tINTEGER: + case IToken.t_false: + case IToken.t_true: + case IToken.tSTRING: + case IToken.tLSTRING: + case IToken.tFLOATINGPT: + case IToken.tCHAR: + case IToken.tAMPER: + case IToken.tDOT: + case IToken.tLPAREN: + case IToken.tMINUS: + case IToken.tSTAR: + case IToken.tPLUS: + case IToken.tNOT: + case IToken.tCOMPL: + return true; + default: + break; + } + return false; + } + + public boolean isOperator() + { + switch( getType() ) + { + case IToken.t_new: + case IToken.t_delete: + case IToken.tPLUS: + case IToken.tMINUS: + case IToken.tSTAR: + case IToken.tDIV: + case IToken.tXOR: + case IToken.tMOD: + case IToken.tAMPER: + case IToken.tBITOR: + case IToken.tCOMPL: + case IToken.tNOT: + case IToken.tASSIGN: + case IToken.tLT: + case IToken.tGT: + case IToken.tPLUSASSIGN: + case IToken.tMINUSASSIGN: + case IToken.tSTARASSIGN: + case IToken.tDIVASSIGN: + case IToken.tMODASSIGN: + case IToken.tBITORASSIGN: + case IToken.tAMPERASSIGN: + case IToken.tXORASSIGN: + case IToken.tSHIFTL: + case IToken.tSHIFTR: + case IToken.tSHIFTLASSIGN: + case IToken.tSHIFTRASSIGN: + case IToken.tEQUAL: + case IToken.tNOTEQUAL: + case IToken.tLTEQUAL: + case IToken.tGTEQUAL: + case IToken.tAND: + case IToken.tOR: + case IToken.tINCR: + case IToken.tDECR: + case IToken.tCOMMA: + case IToken.tARROW: + case IToken.tARROWSTAR: + return true; + default: + return false; + } + } + + public boolean isPointer() + { + return (getType() == IToken.tAMPER || getType() == IToken.tSTAR); + } + + + + public final IToken getNext() { return next; } + public void setNext(IToken t) { next = t; } + +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedExpansionToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedExpansionToken.java new file mode 100644 index 00000000000..5749e015f2d --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedExpansionToken.java @@ -0,0 +1,48 @@ +/********************************************************************** + * Copyright (c) 2002-2004 IBM Canada and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Rational Software - Initial API and implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.token; + +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.internal.core.parser.scanner.ContextStack; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext; + +/** + * @author johnc + */ +public class ImagedExpansionToken extends ImagedToken implements IToken { + + /** + * @param t + * @param contextStack + * @param i + */ + public ImagedExpansionToken(int t, ContextStack contextStack, String i) { + super(t, contextStack, i); + } + + protected int length; + + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.token.SimpleToken#setOffsetAndLength(org.eclipse.cdt.internal.core.parser.scanner.IScannerContext) + */ + protected void setOffsetAndLength(IScannerContext context) { + offset = context.getMacroOffset(); + length = context.getMacroLength(); + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getLength() + */ + public final int getLength() { + return length; + } + +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedToken.java new file mode 100644 index 00000000000..c9177ec8eb7 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedToken.java @@ -0,0 +1,78 @@ +/********************************************************************** + * Copyright (c) 2002-2004 IBM Canada and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Rational Software - Initial API and implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.token; + +import org.eclipse.cdt.internal.core.parser.scanner.ContextStack; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext; + +/** + * @author johnc + */ +public class ImagedToken extends SimpleToken { + + protected String image = null; + + /** + * @param t + * @param contextStack + * @param i + */ + public ImagedToken(int t, ContextStack contextStack, String i) { + super(t, contextStack); + setImage( i ); + setOffsetAndLength(contextStack.getCurrentContext()); + } + + public ImagedToken( int t, String i) { + super( t ); + setImage(i); + } + + + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getImage() + */ + public final String getImage() { + return image; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#setImage(java.lang.String) + */ + public void setImage(String i) { + image = i; + } + + /** + * @param context + */ + protected void setOffsetAndLength(IScannerContext context) { + if( getImage() == null ) return; + offset = context.getOffset() - getImage().length() - context.undoStackSize(); + if( getType() == tSTRING || getType() == tCHAR ) + offset--; + else if( getType() == tLSTRING || getType() == tLCHAR ) + offset -= 2; + } + + public int getLength() { + switch( getType() ) + { + case tSTRING: + case tCHAR: + return getImage().length() + 2; // 'c' is 3 characters, not 1 + case tLSTRING: + case tLCHAR: + return getImage().length() + 3; // L"X" if 4 characters, not 1 + default: + return getImage().length(); + } + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleExpansionToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleExpansionToken.java new file mode 100644 index 00000000000..bf1c8b27f96 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleExpansionToken.java @@ -0,0 +1,48 @@ +/********************************************************************** + * Copyright (c) 2002-2004 IBM Canada and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Rational Software - Initial API and implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.token; + +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.internal.core.parser.scanner.ContextStack; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext; + +/** + * @author johnc + */ +public class SimpleExpansionToken extends SimpleToken implements IToken { + + /** + * @param tokenType + * @param stack + */ + public SimpleExpansionToken(int tokenType, ContextStack stack) { + super( tokenType, stack ); + } + + protected int length; + + + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.token.SimpleToken#setOffsetAndLength(org.eclipse.cdt.internal.core.parser.scanner.IScannerContext) + */ + protected void setOffsetAndLength(IScannerContext context) { + offset = context.getMacroOffset(); + length = context.getMacroLength(); + } + + + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getLength() + */ + public final int getLength() { + return length; + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/Token.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleToken.java similarity index 64% rename from core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/Token.java rename to core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleToken.java index 33b00495287..3d34f984a3c 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/Token.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleToken.java @@ -10,71 +10,45 @@ ******************************************************************************/ package org.eclipse.cdt.internal.core.parser.token; -import org.eclipse.cdt.core.parser.Keywords; import org.eclipse.cdt.core.parser.IScanner; import org.eclipse.cdt.core.parser.IToken; -import org.eclipse.cdt.internal.core.parser.scanner.*; +import org.eclipse.cdt.core.parser.Keywords; +import org.eclipse.cdt.internal.core.parser.scanner.ContextStack; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext; -public class Token implements IToken { - - public Token(int t, String i, IScannerContext context, int lineNumber ) { - setType(t); - setImage(i); - filename = context.getFilename(); - offset = context.getOffset() - getImage().length() - context.undoStackSize(); - this.lineNumber = lineNumber; - macroOffset = context.getMacroOffset(); - macroLength = context.getMacroLength(); - - if( type == tLSTRING || type == tSTRING || type == tCHAR ){ - offset--; - } - } - public Token(int t, IScannerContext context, int lineNumber ) { - setType(t); - setImage(null); - filename = context.getFilename(); - offset = context.getOffset() - getImage().length() - context.undoStackSize(); - this.lineNumber = lineNumber; - macroOffset = context.getMacroOffset(); - macroLength = context.getMacroLength(); - - if( type == tLSTRING || type == tSTRING || type == tCHAR ){ - offset--; - } +public class SimpleToken extends AbstractToken implements IToken { + + public SimpleToken(int t, ContextStack contextStack ) { + super(t,contextStack.getCurrentLineNumber()); + setOffsetAndLength(contextStack.getCurrentContext()); } - public Token(int t) { - setType(t); - setImage(null); - } - public Token(int t, String i) { - setType(t); - setImage(i); - } - - public String toString() + public SimpleToken( int t ) { - return "Token=" + type + " \"" + getImage() + "\" " + filename + ":" + lineNumber + " offset=" + offset; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + super( t ); } - public int type; - public int getType() { return type; } + protected int offset; + + // All the tokens generated by the macro expansion + // will have dimensions (offset and length) equal to the expanding symbol. + public int getOffset() { + return offset; + } - protected String image; + public int getLength() { + return getImage().length(); + } + + /** + * @param context + */ + protected void setOffsetAndLength(IScannerContext context) { + offset = context.getOffset() - getImage().length() - context.undoStackSize(); + } public String getImage() { switch ( getType() ) { - - case IToken.tIDENTIFIER : - case IToken.tINTEGER : - case IToken.tFLOATINGPT : - case IToken.tSTRING : - case IToken.tLSTRING : - case IToken.tCHAR : - case IToken.tLCHAR : - return image; - case IToken.tCOLONCOLON : return "::" ; //$NON-NLS-1$ case IToken.tCOLON : @@ -336,164 +310,17 @@ public class Token implements IToken { default : // we should never get here! - return image; + // assert false : getType(); + return ""; //$NON-NLS-1$ } } - - public String filename; - - protected int offset; - protected int macroOffset = -1; - protected int macroLength = -1; - // All the tokens generated by the macro expansion - // will have dimensions (offset and length) equal to the expanding symbol. - public int getOffset() { return (macroOffset < 0) ? offset : macroOffset; } - - public int getLength() { return (macroLength < 0) ? getImage().length() : macroLength; } - - public int getEndOffset() { return getOffset() + getLength(); } + - - public int getDelta( IToken other ) - { - return other.getOffset() + other.getLength() - getOffset(); - } - - private IToken next; - - private int lineNumber; - public IToken getNext() { return next; } - public void setNext(IToken t) { next = t; } - - public boolean looksLikeExpression() - { - switch( getType() ) - { - case tINTEGER: - case t_false: - case t_true: - case tSTRING: - case tLSTRING: - case tFLOATINGPT: - case tCHAR: - case tAMPER: - case tDOT: - case tLPAREN: - case tMINUS: - case tSTAR: - case tPLUS: - case tNOT: - case tCOMPL: - return true; - default: - break; - } - - - return false; - } - - public boolean isPointer() - { - return (getType() == tAMPER || getType() == tSTAR); - } - - public boolean isOperator() - { - switch( getType() ) - { - case IToken.t_new: - case IToken.t_delete: - case IToken.tPLUS: - case IToken.tMINUS: - case IToken.tSTAR: - case IToken.tDIV: - case IToken.tXOR: - case IToken.tMOD: - case IToken.tAMPER: - case IToken.tBITOR: - case IToken.tCOMPL: - case IToken.tNOT: - case IToken.tASSIGN: - case IToken.tLT: - case IToken.tGT: - case IToken.tPLUSASSIGN: - case IToken.tMINUSASSIGN: - case IToken.tSTARASSIGN: - case IToken.tDIVASSIGN: - case IToken.tMODASSIGN: - case IToken.tBITORASSIGN: - case IToken.tAMPERASSIGN: - case IToken.tXORASSIGN: - case IToken.tSHIFTL: - case IToken.tSHIFTR: - case IToken.tSHIFTLASSIGN: - case IToken.tSHIFTRASSIGN: - case IToken.tEQUAL: - case IToken.tNOTEQUAL: - case IToken.tLTEQUAL: - case IToken.tGTEQUAL: - case IToken.tAND: - case IToken.tOR: - case IToken.tINCR: - case IToken.tDECR: - case IToken.tCOMMA: - case IToken.tARROW: - case IToken.tARROWSTAR: - return true; - default: - return false; - } - } - /* (non-Javadoc) * @see org.eclipse.cdt.core.parser.IToken#setImage() */ public void setImage( String i ) { - image = i; - } - - - /* (non-Javadoc) - * @see java.lang.Object#equals(java.lang.Object) - */ - public boolean equals(Object other) { - if( other == null ) return false; - if( !( other instanceof IToken ) ) - return false; - if( !(((IToken)other).getImage().equals( getImage() ))) - return false; - if( ((IToken)other).getType() != type ) - return false; - return true; - } - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.IToken#setType(int) - */ - public void setType(int i) { - type = i; - } - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.IToken#getLineNumber() - */ - public int getLineNumber() { - return lineNumber; - } - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.IToken#isKeyword() - */ - public boolean isKeywordOrOperator() { - if( type == IToken.tCHAR ) return false; - if( type == IToken.tFLOATINGPT ) return false; - if( type == IToken.tIDENTIFIER ) return false; - if( type == IToken.tINTEGER ) return false; - if( type == IToken.tSTRING ) return false; - if( type == IToken.tLSTRING ) return false; - if( type == IToken.tLCHAR ) return false; - return true; + // do nothing } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java new file mode 100644 index 00000000000..80523e38dc9 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java @@ -0,0 +1,47 @@ +/********************************************************************** + * Copyright (c) 2002-2004 IBM Canada and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Rational Software - Initial API and implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.token; + +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerData; + +/** + * @author johnc + */ +public class TokenFactory { + + public static IToken createToken( int tokenType, IScannerData scannerData ) + { + if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 ) + return new SimpleExpansionToken( tokenType, scannerData.getContextStack() ); + + return new SimpleToken( tokenType, scannerData.getContextStack() ); + } + + /** + * @param type + * @param image + * @param scannerData + * @return + */ + public static IToken createToken(int type, String image, IScannerData scannerData) { + if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 ) + return new ImagedExpansionToken( type, scannerData.getContextStack(), image ); + + return new ImagedToken(type, scannerData.getContextStack(), image ); + } + + + public static IToken createToken( int type, String image ) + { + return new ImagedToken( type, image); + } +}