From 8a4ada65b3ce6fb4228a35db8dc2d9d93e8cd9a1 Mon Sep 17 00:00:00 2001 From: John Camelon Date: Sun, 11 Apr 2004 04:18:24 +0000 Subject: [PATCH]
org.eclipse.cdt.core
  Removed unused methods from IToken.
  Restructured Token implementation to be more memory efficient.
  Tightened up visibility of different members in the implementation.
  Added TokenFactory construct to isolate IToken clients from the particulars of which IToken implementation to instantiate.

org.eclipse.cdt.core.tests
  Updated ScannerTestCase to correspond with changes to IToken and its implementations. --- core/org.eclipse.cdt.core.tests/ChangeLog | 3 + .../core/parser/tests/ScannerTestCase.java | 12 +- .../parser/ChangeLog-parser | 6 + .../org/eclipse/cdt/core/parser/IToken.java | 41 +-- .../core/parser/ContextualParser.java | 6 +- .../core/parser/ExpressionParser.java | 4 +- .../parser/scanner/ObjectMacroDescriptor.java | 6 +- .../internal/core/parser/scanner/Scanner.java | 76 +++--- .../core/parser/token/AbstractToken.java | 173 +++++++++++++ .../parser/token/ImagedExpansionToken.java | 48 ++++ .../core/parser/token/ImagedToken.java | 78 ++++++ .../parser/token/SimpleExpansionToken.java | 48 ++++ .../token/{Token.java => SimpleToken.java} | 235 +++--------------- .../core/parser/token/TokenFactory.java | 47 ++++ 14 files changed, 498 insertions(+), 285 deletions(-) create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/AbstractToken.java create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedExpansionToken.java create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedToken.java create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleExpansionToken.java rename core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/{Token.java => SimpleToken.java} (64%) create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java diff --git a/core/org.eclipse.cdt.core.tests/ChangeLog b/core/org.eclipse.cdt.core.tests/ChangeLog index 153dffda301..3f739b6eb8d 100644 --- a/core/org.eclipse.cdt.core.tests/ChangeLog +++ b/core/org.eclipse.cdt.core.tests/ChangeLog @@ -1,3 +1,6 @@ +2004-04-10 John Camelon + Updated ScannerTestCase to correspond with changes to IToken and its implementations. + 2004-04-09 Andrew Niefer added parser/org/eclipse/cdt/core/parser/tests/CompleteParseASTTemplateTest.testTemplateMemberTemplateDefinition() diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java index 91628dda42d..a72c6e6cc72 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java @@ -25,7 +25,7 @@ import org.eclipse.cdt.core.parser.ParserFactoryError; import org.eclipse.cdt.core.parser.ParserMode; import org.eclipse.cdt.core.parser.ScannerException; import org.eclipse.cdt.core.parser.ast.IASTInclusion; -import org.eclipse.cdt.internal.core.parser.token.Token; +import org.eclipse.cdt.internal.core.parser.token.SimpleToken; /** * @author jcamelon @@ -734,11 +734,11 @@ public class ScannerTestCase extends BaseScannerTest List expansion= descriptor.getTokenizedExpansion(); assertNotNull(parms); assertTrue(expansion.size() == 3); - assertTrue(((Token) expansion.get(0)).type == IToken.tIDENTIFIER); - assertTrue(((Token) expansion.get(0)).getImage().equals("x")); - assertTrue(((Token) expansion.get(1)).type == IToken.tPLUS); - assertTrue(((Token) expansion.get(2)).type == IToken.tINTEGER); - assertTrue(((Token) expansion.get(2)).getImage().equals("1")); + assertTrue(((SimpleToken) expansion.get(0)).getType() == IToken.tIDENTIFIER); + assertTrue(((SimpleToken) expansion.get(0)).getImage().equals("x")); + assertTrue(((SimpleToken) expansion.get(1)).getType() == IToken.tPLUS); + assertTrue(((SimpleToken) expansion.get(2)).getType() == IToken.tINTEGER); + assertTrue(((SimpleToken) expansion.get(2)).getImage().equals("1")); validateIdentifier("y"); validateToken(IToken.tASSIGN); diff --git a/core/org.eclipse.cdt.core/parser/ChangeLog-parser b/core/org.eclipse.cdt.core/parser/ChangeLog-parser index 590a14f43b4..2be9a6a2b85 100644 --- a/core/org.eclipse.cdt.core/parser/ChangeLog-parser +++ b/core/org.eclipse.cdt.core/parser/ChangeLog-parser @@ -1,3 +1,9 @@ +2004-04-10 John Camelon + Removed unused methods from IToken. + Restructured Token implementation to be more memory efficient. + Tightened up visibility of different members in the implementation. + Added TokenFactory construct to isolate Token clients from the particulars of which IToken implementation to instantiate. + 2004-04-10 John Camelon Partial fix for https://bugs.eclipse.org/bugs/show_bug.cgi?id=57898 diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java index 381eed9e945..6b9a37d59ff 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java @@ -16,24 +16,30 @@ package org.eclipse.cdt.core.parser; * */ public interface IToken { - public abstract String toString(); - public abstract int getType(); - public abstract String getImage(); - public void setImage( String i ); - public abstract int getOffset(); - public abstract int getLength(); - public abstract int getEndOffset(); + // getters + public int getType(); + public String getImage(); + public int getOffset(); + public int getLength(); + public int getEndOffset(); // NOTE:if the token spans lines due to escaped newlines then // the line number returned is the last one - public int getLineNumber(); - public abstract int getDelta(IToken other); - public abstract IToken getNext(); - public abstract void setNext(IToken t); - public abstract void setType(int i); - public abstract boolean looksLikeExpression(); - public abstract boolean isPointer(); - public abstract boolean isOperator(); + public int getLineNumber(); + public IToken getNext(); + + // setters + public void setImage( String i ); + public void setNext(IToken t); + public void setType(int i); + + // queries + public boolean looksLikeExpression(); + public boolean isPointer(); + public boolean isOperator(); + public boolean isKeywordOrOperator(); + + // Token types static public final int tIDENTIFIER = 1; @@ -307,9 +313,4 @@ public interface IToken { static public final int tLAST = t_restrict; - /** - * @return - */ - public abstract boolean isKeywordOrOperator(); - } \ No newline at end of file diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ContextualParser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ContextualParser.java index 7c958797d41..3926a563551 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ContextualParser.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ContextualParser.java @@ -26,8 +26,8 @@ import org.eclipse.cdt.core.parser.ast.IASTNode; import org.eclipse.cdt.core.parser.ast.IASTScope; import org.eclipse.cdt.core.parser.ast.IASTCompletionNode.CompletionKind; import org.eclipse.cdt.internal.core.parser.token.KeywordSets; -import org.eclipse.cdt.internal.core.parser.token.Token; import org.eclipse.cdt.internal.core.parser.token.TokenDuple; +import org.eclipse.cdt.internal.core.parser.token.TokenFactory; import org.eclipse.cdt.internal.core.parser.token.KeywordSets.Key; /** @@ -107,7 +107,7 @@ public class ContextualParser extends CompleteParser { setCompletionKind(kind); setCompletionContext(null); setCompletionFunctionName( ); - setCompletionToken( new Token( IToken.tIDENTIFIER, prefix ) ); + setCompletionToken( TokenFactory.createToken( IToken.tIDENTIFIER, prefix ) ); } /** @@ -132,7 +132,7 @@ public class ContextualParser extends CompleteParser { } protected void setCompletionValues(IASTScope scope, CompletionKind kind, Key key, IASTNode node, String prefix) throws EndOfFileException { - setCompletionToken( new Token( IToken.tIDENTIFIER, prefix ) ); + setCompletionToken( TokenFactory.createToken( IToken.tIDENTIFIER, prefix ) ); setCompletionValues(scope, kind, key, node ); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java index 35fc817b714..ebe1fe9fc4f 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java @@ -40,7 +40,7 @@ import org.eclipse.cdt.core.parser.ast.IASTTypeId; import org.eclipse.cdt.core.parser.ast.IASTCompletionNode.CompletionKind; import org.eclipse.cdt.core.parser.ast.IASTExpression.Kind; import org.eclipse.cdt.internal.core.parser.token.KeywordSets; -import org.eclipse.cdt.internal.core.parser.token.Token; +import org.eclipse.cdt.internal.core.parser.token.SimpleToken; import org.eclipse.cdt.internal.core.parser.token.TokenDuple; import org.eclipse.cdt.internal.core.parser.token.KeywordSets.Key; import org.eclipse.cdt.internal.core.parser.util.TraceUtil; @@ -2674,7 +2674,7 @@ public class ExpressionParser implements IExpressionParser { * */ protected void backup(IToken mark) { - currToken = (Token)mark; + currToken = (SimpleToken)mark; lastToken = null; // this is not entirely right ... } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ObjectMacroDescriptor.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ObjectMacroDescriptor.java index 19e99eb9bbf..61225eb3711 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ObjectMacroDescriptor.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ObjectMacroDescriptor.java @@ -14,7 +14,7 @@ import java.util.ArrayList; import java.util.List; import org.eclipse.cdt.core.parser.IMacroDescriptor; -import org.eclipse.cdt.internal.core.parser.token.Token; +import org.eclipse.cdt.core.parser.IToken; /** * @author jcamelon @@ -24,7 +24,7 @@ public class ObjectMacroDescriptor implements IMacroDescriptor { private static final ArrayList EMPTY_LIST = new ArrayList(); private final String expansionSignature; private final String name; - private final Token token; + private final IToken token; public ObjectMacroDescriptor( String name, String expansionSignature ) { @@ -33,7 +33,7 @@ public class ObjectMacroDescriptor implements IMacroDescriptor { token = null; } - public ObjectMacroDescriptor( String name, Token t, String expansionSignature ) + public ObjectMacroDescriptor( String name, IToken t, String expansionSignature ) { this.name = name; this.token = t; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java index c63d769a1dd..d185a54e580 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java @@ -58,7 +58,8 @@ import org.eclipse.cdt.internal.core.parser.IExpressionParser; import org.eclipse.cdt.internal.core.parser.InternalParserUtil; import org.eclipse.cdt.internal.core.parser.ast.ASTCompletionNode; import org.eclipse.cdt.internal.core.parser.token.KeywordSets; -import org.eclipse.cdt.internal.core.parser.token.Token; +import org.eclipse.cdt.internal.core.parser.token.SimpleToken; +import org.eclipse.cdt.internal.core.parser.token.TokenFactory; import org.eclipse.cdt.internal.core.parser.util.TraceUtil; /** @@ -487,25 +488,14 @@ public class Scanner implements IScanner { storageBuffer = null; } - protected IToken newToken(int t, String i, IScannerContext c) { - setCurrentToken(new Token(t, i, c, scannerData.getContextStack().getCurrentLineNumber())); + protected IToken newToken(int t, String i) { + IToken theToken = TokenFactory.createToken( t, i, scannerData ); + setCurrentToken(theToken); return currentToken; } protected IToken newConstantToken(int t) { - setCurrentToken( - new Token( - t, - scannerData.getContextStack().getCurrentContext(), - scannerData.getContextStack().getCurrentLineNumber() - ) - ); - return currentToken; - } - - - protected IToken newToken(int t, String i) { - setCurrentToken(new Token(t, i)); + setCurrentToken( TokenFactory.createToken(t,scannerData)); return currentToken; } @@ -1017,7 +1007,7 @@ public class Scanner implements IScanner { //If the next token is going to be a string as well, we need to concatenate //it with this token. This will be recursive for as many strings as need to be concatenated - IToken returnToken = newToken( type, buff.toString(), scannerData.getContextStack().getCurrentContext()); + IToken returnToken = newToken( type, buff.toString()); IToken next = null; try{ @@ -1177,8 +1167,7 @@ public class Scanner implements IScanner { return newToken( tokenType, - result, - scannerData.getContextStack().getCurrentContext()); + result); } public IToken processPreprocessor() throws ScannerException, EndOfFileException { @@ -1573,7 +1562,7 @@ public class Scanner implements IScanner { if (tokenTypeObject != null) return newConstantToken(((Integer) tokenTypeObject).intValue()); else - return newToken(IToken.tIDENTIFIER, ident, scannerData.getContextStack().getCurrentContext()); + return newToken(IToken.tIDENTIFIER, ident); } /** @@ -2143,7 +2132,7 @@ public class Scanner implements IScanner { c = getChar(true); } - return newToken( type, buffer.toString(), scannerData.getContextStack().getCurrentContext()); + return newToken( type, buffer.toString()); } @@ -2196,7 +2185,7 @@ public class Scanner implements IScanner { if (c != NOCHAR ) { - return newToken( IToken.tSTRING, buff.toString(), scannerData.getContextStack().getCurrentContext()); + return newToken( IToken.tSTRING, buff.toString()); } else { handleProblem( IProblem.SCANNER_UNBOUNDED_STRING, null, beginOffset, false, true ); @@ -2211,13 +2200,13 @@ public class Scanner implements IScanner { return processCharacterLiteral( c, false ); case ',' : if (tokenImage.length() > 0) throw endOfMacroToken; - return newToken(IToken.tCOMMA, ",", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$ + return newToken(IToken.tCOMMA, ","); //$NON-NLS-1$ case '(' : if (tokenImage.length() > 0) throw endOfMacroToken; - return newToken(IToken.tLPAREN, "(", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$ + return newToken(IToken.tLPAREN, "("); //$NON-NLS-1$ case ')' : if (tokenImage.length() > 0) throw endOfMacroToken; - return newToken(IToken.tRPAREN, ")", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$ + return newToken(IToken.tRPAREN, ")"); //$NON-NLS-1$ case '/' : if (tokenImage.length() > 0) throw endOfMacroToken; c = getChar(); @@ -2247,7 +2236,7 @@ public class Scanner implements IScanner { // return completed token if (tokenImage.length() > 0) { - return newToken(IToken.tIDENTIFIER, tokenImage.toString(), scannerData.getContextStack().getCurrentContext()); + return newToken(IToken.tIDENTIFIER, tokenImage.toString()); } // we're done @@ -2678,16 +2667,9 @@ public class Scanner implements IScanner { } protected IMacroDescriptor createObjectMacroDescriptor(String key, String value ) { - Token t = null; + IToken t = null; if( !value.trim().equals( "" ) ) //$NON-NLS-1$ - { - t = new Token( - IToken.tIDENTIFIER, - value, - scannerData.getContextStack().getCurrentContext(), - scannerData.getContextStack().getCurrentLineNumber() - ); - } + t = TokenFactory.createToken( IToken.tIDENTIFIER, value, scannerData ); return new ObjectMacroDescriptor( key, t, @@ -2883,7 +2865,7 @@ public class Scanner implements IScanner { tokenizer.setThrowExceptionOnBadCharacterRead(false); Vector parameterValues = new Vector(); - Token t = null; + SimpleToken t = null; StringBuffer buffer = new StringBuffer(); boolean space = false; int nParen = 0; @@ -2896,12 +2878,12 @@ public class Scanner implements IScanner { } if (c != NOCHAR) tokenizer.ungetChar(c); - t = (Token)(forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false)); - if (t.type == IToken.tLPAREN) { + t = (SimpleToken)(forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false)); + if (t.getType() == IToken.tLPAREN) { nParen++; - } else if (t.type == IToken.tRPAREN) { + } else if (t.getType() == IToken.tRPAREN) { nParen--; - } else if (t.type == IToken.tCOMMA && nParen == 0) { + } else if (t.getType() == IToken.tCOMMA && nParen == 0) { parameterValues.add(buffer.toString()); buffer = new StringBuffer(); space = false; @@ -2911,7 +2893,7 @@ public class Scanner implements IScanner { if (space) buffer.append( ' ' ); - switch (t.type) { + switch (t.getType()) { case IToken.tSTRING : buffer.append('\"'); buffer.append(t.getImage()); @@ -3002,7 +2984,7 @@ public class Scanner implements IScanner { Vector parameterValues = getMacroParameters(betweenTheBrackets, false); Vector parameterValuesForStringizing = getMacroParameters(betweenTheBrackets, true); - Token t = null; + SimpleToken t = null; // create a string that represents what needs to be tokenized buffer = new StringBuffer(); @@ -3019,8 +3001,8 @@ public class Scanner implements IScanner { int numberOfTokens = tokens.size(); for (int i = 0; i < numberOfTokens; ++i) { - t = (Token) tokens.get(i); - if (t.type == IToken.tIDENTIFIER) { + t = (SimpleToken) tokens.get(i); + if (t.getType() == IToken.tIDENTIFIER) { // is this identifier in the parameterNames // list? @@ -3033,10 +3015,10 @@ public class Scanner implements IScanner { buffer.append( (String) parameterValues.elementAt(index) ); } - } else if (t.type == tPOUND) { + } else if (t.getType() == tPOUND) { //next token should be a parameter which needs to be turned into //a string literal - t = (Token) tokens.get( ++i ); + t = (SimpleToken) tokens.get( ++i ); int index = parameterNames.indexOf(t.getImage()); if( index == -1 ){ handleProblem( IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, expansion.getName(), getCurrentOffset(), false, true ); @@ -3067,7 +3049,7 @@ public class Scanner implements IScanner { buffer.append('\"'); } } else { - switch( t.type ) + switch( t.getType() ) { case IToken.tSTRING: buffer.append('\"'); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/AbstractToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/AbstractToken.java new file mode 100644 index 00000000000..1b4e5829eeb --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/AbstractToken.java @@ -0,0 +1,173 @@ +/********************************************************************** + * Copyright (c) 2002-2004 IBM Canada and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Rational Software - Initial API and implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.token; + +import org.eclipse.cdt.core.parser.IToken; + + +/** + * @author johnc + */ +public abstract class AbstractToken { + + public AbstractToken( int type, int lineNumber ) + { + setType( type ); + this.lineNumber = lineNumber; + } + + public AbstractToken( int type ) + { + setType( type ); + } + + public String toString() { + return "Token=" + getType() + " \"" + getImage() + " @ line:" + getLineNumber() + " offset=" + getOffset(); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ + } + + public abstract String getImage(); + public abstract int getOffset(); + public abstract int getLength(); + + public int getType() { return type; } + + public void setType(int i) { + type = i; + } + + public int getLineNumber() { + return lineNumber; + } + + public int getEndOffset() { return getOffset() + getLength(); } + + protected int type; + protected int lineNumber = 1; + protected IToken next = null; + + /* (non-Javadoc) + * @see java.lang.Object#equals(java.lang.Object) + */ + public boolean equals(Object other) { + if( other == null ) return false; + if( !( other instanceof IToken ) ) + return false; + if( ((IToken)other).getType() != getType() ) + return false; + if( !(((IToken)other).getImage().equals( getImage() ))) + return false; + return true; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#isKeyword() + */ + public boolean isKeywordOrOperator() { + switch( getType() ) + { + case IToken.tCHAR: + case IToken.tFLOATINGPT: + case IToken.tIDENTIFIER: + case IToken.tINTEGER: + case IToken.tSTRING: + case IToken.tLSTRING: + case IToken.tLCHAR: + return false; + default: + return true; + } + } + + public boolean looksLikeExpression() + { + switch( getType() ) + { + case IToken.tINTEGER: + case IToken.t_false: + case IToken.t_true: + case IToken.tSTRING: + case IToken.tLSTRING: + case IToken.tFLOATINGPT: + case IToken.tCHAR: + case IToken.tAMPER: + case IToken.tDOT: + case IToken.tLPAREN: + case IToken.tMINUS: + case IToken.tSTAR: + case IToken.tPLUS: + case IToken.tNOT: + case IToken.tCOMPL: + return true; + default: + break; + } + return false; + } + + public boolean isOperator() + { + switch( getType() ) + { + case IToken.t_new: + case IToken.t_delete: + case IToken.tPLUS: + case IToken.tMINUS: + case IToken.tSTAR: + case IToken.tDIV: + case IToken.tXOR: + case IToken.tMOD: + case IToken.tAMPER: + case IToken.tBITOR: + case IToken.tCOMPL: + case IToken.tNOT: + case IToken.tASSIGN: + case IToken.tLT: + case IToken.tGT: + case IToken.tPLUSASSIGN: + case IToken.tMINUSASSIGN: + case IToken.tSTARASSIGN: + case IToken.tDIVASSIGN: + case IToken.tMODASSIGN: + case IToken.tBITORASSIGN: + case IToken.tAMPERASSIGN: + case IToken.tXORASSIGN: + case IToken.tSHIFTL: + case IToken.tSHIFTR: + case IToken.tSHIFTLASSIGN: + case IToken.tSHIFTRASSIGN: + case IToken.tEQUAL: + case IToken.tNOTEQUAL: + case IToken.tLTEQUAL: + case IToken.tGTEQUAL: + case IToken.tAND: + case IToken.tOR: + case IToken.tINCR: + case IToken.tDECR: + case IToken.tCOMMA: + case IToken.tARROW: + case IToken.tARROWSTAR: + return true; + default: + return false; + } + } + + public boolean isPointer() + { + return (getType() == IToken.tAMPER || getType() == IToken.tSTAR); + } + + + + public final IToken getNext() { return next; } + public void setNext(IToken t) { next = t; } + +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedExpansionToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedExpansionToken.java new file mode 100644 index 00000000000..5749e015f2d --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedExpansionToken.java @@ -0,0 +1,48 @@ +/********************************************************************** + * Copyright (c) 2002-2004 IBM Canada and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Rational Software - Initial API and implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.token; + +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.internal.core.parser.scanner.ContextStack; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext; + +/** + * @author johnc + */ +public class ImagedExpansionToken extends ImagedToken implements IToken { + + /** + * @param t + * @param contextStack + * @param i + */ + public ImagedExpansionToken(int t, ContextStack contextStack, String i) { + super(t, contextStack, i); + } + + protected int length; + + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.token.SimpleToken#setOffsetAndLength(org.eclipse.cdt.internal.core.parser.scanner.IScannerContext) + */ + protected void setOffsetAndLength(IScannerContext context) { + offset = context.getMacroOffset(); + length = context.getMacroLength(); + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getLength() + */ + public final int getLength() { + return length; + } + +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedToken.java new file mode 100644 index 00000000000..c9177ec8eb7 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedToken.java @@ -0,0 +1,78 @@ +/********************************************************************** + * Copyright (c) 2002-2004 IBM Canada and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Rational Software - Initial API and implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.token; + +import org.eclipse.cdt.internal.core.parser.scanner.ContextStack; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext; + +/** + * @author johnc + */ +public class ImagedToken extends SimpleToken { + + protected String image = null; + + /** + * @param t + * @param contextStack + * @param i + */ + public ImagedToken(int t, ContextStack contextStack, String i) { + super(t, contextStack); + setImage( i ); + setOffsetAndLength(contextStack.getCurrentContext()); + } + + public ImagedToken( int t, String i) { + super( t ); + setImage(i); + } + + + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getImage() + */ + public final String getImage() { + return image; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#setImage(java.lang.String) + */ + public void setImage(String i) { + image = i; + } + + /** + * @param context + */ + protected void setOffsetAndLength(IScannerContext context) { + if( getImage() == null ) return; + offset = context.getOffset() - getImage().length() - context.undoStackSize(); + if( getType() == tSTRING || getType() == tCHAR ) + offset--; + else if( getType() == tLSTRING || getType() == tLCHAR ) + offset -= 2; + } + + public int getLength() { + switch( getType() ) + { + case tSTRING: + case tCHAR: + return getImage().length() + 2; // 'c' is 3 characters, not 1 + case tLSTRING: + case tLCHAR: + return getImage().length() + 3; // L"X" if 4 characters, not 1 + default: + return getImage().length(); + } + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleExpansionToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleExpansionToken.java new file mode 100644 index 00000000000..bf1c8b27f96 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleExpansionToken.java @@ -0,0 +1,48 @@ +/********************************************************************** + * Copyright (c) 2002-2004 IBM Canada and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Rational Software - Initial API and implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.token; + +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.internal.core.parser.scanner.ContextStack; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext; + +/** + * @author johnc + */ +public class SimpleExpansionToken extends SimpleToken implements IToken { + + /** + * @param tokenType + * @param stack + */ + public SimpleExpansionToken(int tokenType, ContextStack stack) { + super( tokenType, stack ); + } + + protected int length; + + + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.token.SimpleToken#setOffsetAndLength(org.eclipse.cdt.internal.core.parser.scanner.IScannerContext) + */ + protected void setOffsetAndLength(IScannerContext context) { + offset = context.getMacroOffset(); + length = context.getMacroLength(); + } + + + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getLength() + */ + public final int getLength() { + return length; + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/Token.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleToken.java similarity index 64% rename from core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/Token.java rename to core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleToken.java index 33b00495287..3d34f984a3c 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/Token.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleToken.java @@ -10,71 +10,45 @@ ******************************************************************************/ package org.eclipse.cdt.internal.core.parser.token; -import org.eclipse.cdt.core.parser.Keywords; import org.eclipse.cdt.core.parser.IScanner; import org.eclipse.cdt.core.parser.IToken; -import org.eclipse.cdt.internal.core.parser.scanner.*; +import org.eclipse.cdt.core.parser.Keywords; +import org.eclipse.cdt.internal.core.parser.scanner.ContextStack; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext; -public class Token implements IToken { - - public Token(int t, String i, IScannerContext context, int lineNumber ) { - setType(t); - setImage(i); - filename = context.getFilename(); - offset = context.getOffset() - getImage().length() - context.undoStackSize(); - this.lineNumber = lineNumber; - macroOffset = context.getMacroOffset(); - macroLength = context.getMacroLength(); - - if( type == tLSTRING || type == tSTRING || type == tCHAR ){ - offset--; - } - } - public Token(int t, IScannerContext context, int lineNumber ) { - setType(t); - setImage(null); - filename = context.getFilename(); - offset = context.getOffset() - getImage().length() - context.undoStackSize(); - this.lineNumber = lineNumber; - macroOffset = context.getMacroOffset(); - macroLength = context.getMacroLength(); - - if( type == tLSTRING || type == tSTRING || type == tCHAR ){ - offset--; - } +public class SimpleToken extends AbstractToken implements IToken { + + public SimpleToken(int t, ContextStack contextStack ) { + super(t,contextStack.getCurrentLineNumber()); + setOffsetAndLength(contextStack.getCurrentContext()); } - public Token(int t) { - setType(t); - setImage(null); - } - public Token(int t, String i) { - setType(t); - setImage(i); - } - - public String toString() + public SimpleToken( int t ) { - return "Token=" + type + " \"" + getImage() + "\" " + filename + ":" + lineNumber + " offset=" + offset; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + super( t ); } - public int type; - public int getType() { return type; } + protected int offset; + + // All the tokens generated by the macro expansion + // will have dimensions (offset and length) equal to the expanding symbol. + public int getOffset() { + return offset; + } - protected String image; + public int getLength() { + return getImage().length(); + } + + /** + * @param context + */ + protected void setOffsetAndLength(IScannerContext context) { + offset = context.getOffset() - getImage().length() - context.undoStackSize(); + } public String getImage() { switch ( getType() ) { - - case IToken.tIDENTIFIER : - case IToken.tINTEGER : - case IToken.tFLOATINGPT : - case IToken.tSTRING : - case IToken.tLSTRING : - case IToken.tCHAR : - case IToken.tLCHAR : - return image; - case IToken.tCOLONCOLON : return "::" ; //$NON-NLS-1$ case IToken.tCOLON : @@ -336,164 +310,17 @@ public class Token implements IToken { default : // we should never get here! - return image; + // assert false : getType(); + return ""; //$NON-NLS-1$ } } - - public String filename; - - protected int offset; - protected int macroOffset = -1; - protected int macroLength = -1; - // All the tokens generated by the macro expansion - // will have dimensions (offset and length) equal to the expanding symbol. - public int getOffset() { return (macroOffset < 0) ? offset : macroOffset; } - - public int getLength() { return (macroLength < 0) ? getImage().length() : macroLength; } - - public int getEndOffset() { return getOffset() + getLength(); } + - - public int getDelta( IToken other ) - { - return other.getOffset() + other.getLength() - getOffset(); - } - - private IToken next; - - private int lineNumber; - public IToken getNext() { return next; } - public void setNext(IToken t) { next = t; } - - public boolean looksLikeExpression() - { - switch( getType() ) - { - case tINTEGER: - case t_false: - case t_true: - case tSTRING: - case tLSTRING: - case tFLOATINGPT: - case tCHAR: - case tAMPER: - case tDOT: - case tLPAREN: - case tMINUS: - case tSTAR: - case tPLUS: - case tNOT: - case tCOMPL: - return true; - default: - break; - } - - - return false; - } - - public boolean isPointer() - { - return (getType() == tAMPER || getType() == tSTAR); - } - - public boolean isOperator() - { - switch( getType() ) - { - case IToken.t_new: - case IToken.t_delete: - case IToken.tPLUS: - case IToken.tMINUS: - case IToken.tSTAR: - case IToken.tDIV: - case IToken.tXOR: - case IToken.tMOD: - case IToken.tAMPER: - case IToken.tBITOR: - case IToken.tCOMPL: - case IToken.tNOT: - case IToken.tASSIGN: - case IToken.tLT: - case IToken.tGT: - case IToken.tPLUSASSIGN: - case IToken.tMINUSASSIGN: - case IToken.tSTARASSIGN: - case IToken.tDIVASSIGN: - case IToken.tMODASSIGN: - case IToken.tBITORASSIGN: - case IToken.tAMPERASSIGN: - case IToken.tXORASSIGN: - case IToken.tSHIFTL: - case IToken.tSHIFTR: - case IToken.tSHIFTLASSIGN: - case IToken.tSHIFTRASSIGN: - case IToken.tEQUAL: - case IToken.tNOTEQUAL: - case IToken.tLTEQUAL: - case IToken.tGTEQUAL: - case IToken.tAND: - case IToken.tOR: - case IToken.tINCR: - case IToken.tDECR: - case IToken.tCOMMA: - case IToken.tARROW: - case IToken.tARROWSTAR: - return true; - default: - return false; - } - } - /* (non-Javadoc) * @see org.eclipse.cdt.core.parser.IToken#setImage() */ public void setImage( String i ) { - image = i; - } - - - /* (non-Javadoc) - * @see java.lang.Object#equals(java.lang.Object) - */ - public boolean equals(Object other) { - if( other == null ) return false; - if( !( other instanceof IToken ) ) - return false; - if( !(((IToken)other).getImage().equals( getImage() ))) - return false; - if( ((IToken)other).getType() != type ) - return false; - return true; - } - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.IToken#setType(int) - */ - public void setType(int i) { - type = i; - } - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.IToken#getLineNumber() - */ - public int getLineNumber() { - return lineNumber; - } - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.IToken#isKeyword() - */ - public boolean isKeywordOrOperator() { - if( type == IToken.tCHAR ) return false; - if( type == IToken.tFLOATINGPT ) return false; - if( type == IToken.tIDENTIFIER ) return false; - if( type == IToken.tINTEGER ) return false; - if( type == IToken.tSTRING ) return false; - if( type == IToken.tLSTRING ) return false; - if( type == IToken.tLCHAR ) return false; - return true; + // do nothing } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java new file mode 100644 index 00000000000..80523e38dc9 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java @@ -0,0 +1,47 @@ +/********************************************************************** + * Copyright (c) 2002-2004 IBM Canada and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Rational Software - Initial API and implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.token; + +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerData; + +/** + * @author johnc + */ +public class TokenFactory { + + public static IToken createToken( int tokenType, IScannerData scannerData ) + { + if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 ) + return new SimpleExpansionToken( tokenType, scannerData.getContextStack() ); + + return new SimpleToken( tokenType, scannerData.getContextStack() ); + } + + /** + * @param type + * @param image + * @param scannerData + * @return + */ + public static IToken createToken(int type, String image, IScannerData scannerData) { + if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 ) + return new ImagedExpansionToken( type, scannerData.getContextStack(), image ); + + return new ImagedToken(type, scannerData.getContextStack(), image ); + } + + + public static IToken createToken( int type, String image ) + { + return new ImagedToken( type, image); + } +}