mirrors/cdt
1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-04-29 19:45:01 +02:00
Code Issues Wiki Activity

<BR>org.eclipse.cdt.core<BR>

Browse source 
&nbsp;&nbsp;Removed unused methods from IToken.<BR>
&nbsp;&nbsp;Restructured Token implementation to be more memory efficient.<BR>
&nbsp;&nbsp;Tightened up visibility of different members in the implementation.<BR>
&nbsp;&nbsp;Added TokenFactory construct to isolate IToken clients from the particulars of which IToken implementation to instantiate.<BR><BR>
org.eclipse.cdt.core.tests<BR>
&nbsp;&nbsp;Updated ScannerTestCase to correspond with changes to IToken and its implementations.
This commit is contained in:
John Camelon 2004-04-11 04:18:24 +00:00
parent 5f7bf6d199
commit 8a4ada65b3
14 changed files with 498 additions and 285 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
core/org.eclipse.cdt.core.tests/ChangeLog
View file

@ -1,3 +1,6 @@
2004-04-10 John Camelon
Updated ScannerTestCase to correspond with changes to IToken and its implementations.
2004-04-09 Andrew Niefer
added parser/org/eclipse/cdt/core/parser/tests/CompleteParseASTTemplateTest.testTemplateMemberTemplateDefinition()

12
core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java
View file

@ -25,7 +25,7 @@ import org.eclipse.cdt.core.parser.ParserFactoryError;
import org.eclipse.cdt.core.parser.ParserMode;
import org.eclipse.cdt.core.parser.ScannerException;
import org.eclipse.cdt.core.parser.ast.IASTInclusion;
import org.eclipse.cdt.internal.core.parser.token.Token;
import org.eclipse.cdt.internal.core.parser.token.SimpleToken;
/**
* @author jcamelon
@ -734,11 +734,11 @@ public class ScannerTestCase extends BaseScannerTest
List expansion= descriptor.getTokenizedExpansion();
assertNotNull(parms);
assertTrue(expansion.size() == 3);
assertTrue(((Token) expansion.get(0)).type == IToken.tIDENTIFIER);
assertTrue(((Token) expansion.get(0)).getImage().equals("x"));
assertTrue(((Token) expansion.get(1)).type == IToken.tPLUS);
assertTrue(((Token) expansion.get(2)).type == IToken.tINTEGER);
assertTrue(((Token) expansion.get(2)).getImage().equals("1"));
assertTrue(((SimpleToken) expansion.get(0)).getType() == IToken.tIDENTIFIER);
assertTrue(((SimpleToken) expansion.get(0)).getImage().equals("x"));
assertTrue(((SimpleToken) expansion.get(1)).getType() == IToken.tPLUS);
assertTrue(((SimpleToken) expansion.get(2)).getType() == IToken.tINTEGER);
assertTrue(((SimpleToken) expansion.get(2)).getImage().equals("1"));
validateIdentifier("y");
validateToken(IToken.tASSIGN);

6
core/org.eclipse.cdt.core/parser/ChangeLog-parser
View file

@ -1,3 +1,9 @@
2004-04-10 John Camelon
Removed unused methods from IToken.
Restructured Token implementation to be more memory efficient.
Tightened up visibility of different members in the implementation.
Added TokenFactory construct to isolate Token clients from the particulars of which IToken implementation to instantiate.
2004-04-10 John Camelon
Partial fix for https://bugs.eclipse.org/bugs/show_bug.cgi?id=57898

39
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java
View file

@ -16,24 +16,30 @@ package org.eclipse.cdt.core.parser;
*
*/
public interface IToken {
public abstract String toString();
public abstract int getType();
public abstract String getImage();
public void setImage( String i );
public abstract int getOffset();
public abstract int getLength();
public abstract int getEndOffset();
// getters
public int getType();
public String getImage();
public int getOffset();
public int getLength();
public int getEndOffset();
// NOTE:if the token spans lines due to escaped newlines then
// the line number returned is the last one
public int getLineNumber();
public abstract int getDelta(IToken other);
public abstract IToken getNext();
public abstract void setNext(IToken t);
public abstract void setType(int i);
public abstract boolean looksLikeExpression();
public abstract boolean isPointer();
public abstract boolean isOperator();
public IToken getNext();
// setters
public void setImage( String i );
public void setNext(IToken t);
public void setType(int i);
// queries
public boolean looksLikeExpression();
public boolean isPointer();
public boolean isOperator();
public boolean isKeywordOrOperator();
// Token types
static public final int tIDENTIFIER = 1;
@ -307,9 +313,4 @@ public interface IToken {
static public final int tLAST = t_restrict;
/**
* @return
*/
public abstract boolean isKeywordOrOperator();
}

6
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ContextualParser.java
View file

@ -26,8 +26,8 @@ import org.eclipse.cdt.core.parser.ast.IASTNode;
import org.eclipse.cdt.core.parser.ast.IASTScope;
import org.eclipse.cdt.core.parser.ast.IASTCompletionNode.CompletionKind;
import org.eclipse.cdt.internal.core.parser.token.KeywordSets;
import org.eclipse.cdt.internal.core.parser.token.Token;
import org.eclipse.cdt.internal.core.parser.token.TokenDuple;
import org.eclipse.cdt.internal.core.parser.token.TokenFactory;
import org.eclipse.cdt.internal.core.parser.token.KeywordSets.Key;
/**
@ -107,7 +107,7 @@ public class ContextualParser extends CompleteParser {
setCompletionKind(kind);
setCompletionContext(null);
setCompletionFunctionName( );
setCompletionToken( new Token( IToken.tIDENTIFIER, prefix ) );
setCompletionToken( TokenFactory.createToken( IToken.tIDENTIFIER, prefix ) );
}
/**
@ -132,7 +132,7 @@ public class ContextualParser extends CompleteParser {
}
protected void setCompletionValues(IASTScope scope, CompletionKind kind, Key key, IASTNode node, String prefix) throws EndOfFileException {
setCompletionToken( new Token( IToken.tIDENTIFIER, prefix ) );
setCompletionToken( TokenFactory.createToken( IToken.tIDENTIFIER, prefix ) );
setCompletionValues(scope, kind, key, node );
}

4
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ExpressionParser.java
View file

@ -40,7 +40,7 @@ import org.eclipse.cdt.core.parser.ast.IASTTypeId;
import org.eclipse.cdt.core.parser.ast.IASTCompletionNode.CompletionKind;
import org.eclipse.cdt.core.parser.ast.IASTExpression.Kind;
import org.eclipse.cdt.internal.core.parser.token.KeywordSets;
import org.eclipse.cdt.internal.core.parser.token.Token;
import org.eclipse.cdt.internal.core.parser.token.SimpleToken;
import org.eclipse.cdt.internal.core.parser.token.TokenDuple;
import org.eclipse.cdt.internal.core.parser.token.KeywordSets.Key;
import org.eclipse.cdt.internal.core.parser.util.TraceUtil;
@ -2674,7 +2674,7 @@ public class ExpressionParser implements IExpressionParser {
*
*/
protected void backup(IToken mark) {
currToken = (Token)mark;
currToken = (SimpleToken)mark;
lastToken = null; // this is not entirely right ...
}

6
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ObjectMacroDescriptor.java
View file

@ -14,7 +14,7 @@ import java.util.ArrayList;
import java.util.List;
import org.eclipse.cdt.core.parser.IMacroDescriptor;
import org.eclipse.cdt.internal.core.parser.token.Token;
import org.eclipse.cdt.core.parser.IToken;
/**
* @author jcamelon
@ -24,7 +24,7 @@ public class ObjectMacroDescriptor implements IMacroDescriptor {
private static final ArrayList EMPTY_LIST = new ArrayList();
private final String expansionSignature;
private final String name;
private final Token token;
private final IToken token;
public ObjectMacroDescriptor( String name, String expansionSignature )
{
@ -33,7 +33,7 @@ public class ObjectMacroDescriptor implements IMacroDescriptor {
token = null;
}
public ObjectMacroDescriptor( String name, Token t, String expansionSignature )
public ObjectMacroDescriptor( String name, IToken t, String expansionSignature )
{
this.name = name;
this.token = t;

76
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java
View file

@ -58,7 +58,8 @@ import org.eclipse.cdt.internal.core.parser.IExpressionParser;
import org.eclipse.cdt.internal.core.parser.InternalParserUtil;
import org.eclipse.cdt.internal.core.parser.ast.ASTCompletionNode;
import org.eclipse.cdt.internal.core.parser.token.KeywordSets;
import org.eclipse.cdt.internal.core.parser.token.Token;
import org.eclipse.cdt.internal.core.parser.token.SimpleToken;
import org.eclipse.cdt.internal.core.parser.token.TokenFactory;
import org.eclipse.cdt.internal.core.parser.util.TraceUtil;
/**
@ -487,25 +488,14 @@ public class Scanner implements IScanner {
storageBuffer = null;
}
protected IToken newToken(int t, String i, IScannerContext c) {
setCurrentToken(new Token(t, i, c, scannerData.getContextStack().getCurrentLineNumber()));
protected IToken newToken(int t, String i) {
IToken theToken = TokenFactory.createToken( t, i, scannerData );
setCurrentToken(theToken);
return currentToken;
}
protected IToken newConstantToken(int t) {
setCurrentToken(
new Token(
t,
scannerData.getContextStack().getCurrentContext(),
scannerData.getContextStack().getCurrentLineNumber()
)
);
return currentToken;
}
protected IToken newToken(int t, String i) {
setCurrentToken(new Token(t, i));
setCurrentToken( TokenFactory.createToken(t,scannerData));
return currentToken;
}
@ -1017,7 +1007,7 @@ public class Scanner implements IScanner {
//If the next token is going to be a string as well, we need to concatenate
//it with this token. This will be recursive for as many strings as need to be concatenated
IToken returnToken = newToken( type, buff.toString(), scannerData.getContextStack().getCurrentContext());
IToken returnToken = newToken( type, buff.toString());
IToken next = null;
try{
@ -1177,8 +1167,7 @@ public class Scanner implements IScanner {
return newToken(
tokenType,
result,
scannerData.getContextStack().getCurrentContext());
result);
}
public IToken processPreprocessor() throws ScannerException, EndOfFileException
{
@ -1573,7 +1562,7 @@ public class Scanner implements IScanner {
if (tokenTypeObject != null)
return newConstantToken(((Integer) tokenTypeObject).intValue());
else
return newToken(IToken.tIDENTIFIER, ident, scannerData.getContextStack().getCurrentContext());
return newToken(IToken.tIDENTIFIER, ident);
}
/**
@ -2143,7 +2132,7 @@ public class Scanner implements IScanner {
c = getChar(true);
}
return newToken( type, buffer.toString(), scannerData.getContextStack().getCurrentContext());
return newToken( type, buffer.toString());
}
@ -2196,7 +2185,7 @@ public class Scanner implements IScanner {
if (c != NOCHAR )
{
return newToken( IToken.tSTRING, buff.toString(), scannerData.getContextStack().getCurrentContext());
return newToken( IToken.tSTRING, buff.toString());
} else {
handleProblem( IProblem.SCANNER_UNBOUNDED_STRING, null, beginOffset, false, true );
@ -2211,13 +2200,13 @@ public class Scanner implements IScanner {
return processCharacterLiteral( c, false );
case ',' :
if (tokenImage.length() > 0) throw endOfMacroToken;
return newToken(IToken.tCOMMA, ",", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$
return newToken(IToken.tCOMMA, ","); //$NON-NLS-1$
case '(' :
if (tokenImage.length() > 0) throw endOfMacroToken;
return newToken(IToken.tLPAREN, "(", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$
return newToken(IToken.tLPAREN, "("); //$NON-NLS-1$
case ')' :
if (tokenImage.length() > 0) throw endOfMacroToken;
return newToken(IToken.tRPAREN, ")", scannerData.getContextStack().getCurrentContext()); //$NON-NLS-1$
return newToken(IToken.tRPAREN, ")"); //$NON-NLS-1$
case '/' :
if (tokenImage.length() > 0) throw endOfMacroToken;
c = getChar();
@ -2247,7 +2236,7 @@ public class Scanner implements IScanner {
// return completed token
if (tokenImage.length() > 0) {
return newToken(IToken.tIDENTIFIER, tokenImage.toString(), scannerData.getContextStack().getCurrentContext());
return newToken(IToken.tIDENTIFIER, tokenImage.toString());
}
// we're done
@ -2678,16 +2667,9 @@ public class Scanner implements IScanner {
}
protected IMacroDescriptor createObjectMacroDescriptor(String key, String value ) {
Token t = null;
IToken t = null;
if( !value.trim().equals( "" ) ) //$NON-NLS-1$
{
t = new Token(
IToken.tIDENTIFIER,
value,
scannerData.getContextStack().getCurrentContext(),
scannerData.getContextStack().getCurrentLineNumber()
);
}
t = TokenFactory.createToken( IToken.tIDENTIFIER, value, scannerData );
return new ObjectMacroDescriptor( key,
t,
@ -2883,7 +2865,7 @@ public class Scanner implements IScanner {
tokenizer.setThrowExceptionOnBadCharacterRead(false);
Vector parameterValues = new Vector();
Token t = null;
SimpleToken t = null;
StringBuffer buffer = new StringBuffer();
boolean space = false;
int nParen = 0;
@ -2896,12 +2878,12 @@ public class Scanner implements IScanner {
}
if (c != NOCHAR) tokenizer.ungetChar(c);
t = (Token)(forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false));
if (t.type == IToken.tLPAREN) {
t = (SimpleToken)(forStringizing ? tokenizer.nextTokenForStringizing() : tokenizer.nextToken(false));
if (t.getType() == IToken.tLPAREN) {
nParen++;
} else if (t.type == IToken.tRPAREN) {
} else if (t.getType() == IToken.tRPAREN) {
nParen--;
} else if (t.type == IToken.tCOMMA && nParen == 0) {
} else if (t.getType() == IToken.tCOMMA && nParen == 0) {
parameterValues.add(buffer.toString());
buffer = new StringBuffer();
space = false;
@ -2911,7 +2893,7 @@ public class Scanner implements IScanner {
if (space)
buffer.append( ' ' );
switch (t.type) {
switch (t.getType()) {
case IToken.tSTRING :
buffer.append('\"');
buffer.append(t.getImage());
@ -3002,7 +2984,7 @@ public class Scanner implements IScanner {
Vector parameterValues = getMacroParameters(betweenTheBrackets, false);
Vector parameterValuesForStringizing = getMacroParameters(betweenTheBrackets, true);
Token t = null;
SimpleToken t = null;
// create a string that represents what needs to be tokenized
buffer = new StringBuffer();
@ -3019,8 +3001,8 @@ public class Scanner implements IScanner {
int numberOfTokens = tokens.size();
for (int i = 0; i < numberOfTokens; ++i) {
t = (Token) tokens.get(i);
if (t.type == IToken.tIDENTIFIER) {
t = (SimpleToken) tokens.get(i);
if (t.getType() == IToken.tIDENTIFIER) {
// is this identifier in the parameterNames
// list?
@ -3033,10 +3015,10 @@ public class Scanner implements IScanner {
buffer.append(
(String) parameterValues.elementAt(index) );
}
} else if (t.type == tPOUND) {
} else if (t.getType() == tPOUND) {
//next token should be a parameter which needs to be turned into
//a string literal
t = (Token) tokens.get( ++i );
t = (SimpleToken) tokens.get( ++i );
int index = parameterNames.indexOf(t.getImage());
if( index == -1 ){
handleProblem( IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, expansion.getName(), getCurrentOffset(), false, true );
@ -3067,7 +3049,7 @@ public class Scanner implements IScanner {
buffer.append('\"');
}
} else {
switch( t.type )
switch( t.getType() )
{
case IToken.tSTRING:
buffer.append('\"');

173
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/AbstractToken.java Normal file
View file

@ -0,0 +1,173 @@
/**********************************************************************
* Copyright (c) 2002-2004 IBM Canada and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Common Public License v0.5
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/cpl-v05.html
*
* Contributors:
* IBM Rational Software - Initial API and implementation
***********************************************************************/
package org.eclipse.cdt.internal.core.parser.token;
import org.eclipse.cdt.core.parser.IToken;
/**
* @author johnc
*/
public abstract class AbstractToken {
public AbstractToken( int type, int lineNumber )
{
setType( type );
this.lineNumber = lineNumber;
}
public AbstractToken( int type )
{
setType( type );
}
public String toString() {
return "Token=" + getType() + " \"" + getImage() + " @ line:" + getLineNumber() + " offset=" + getOffset(); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
}
public abstract String getImage();
public abstract int getOffset();
public abstract int getLength();
public int getType() { return type; }
public void setType(int i) {
type = i;
}
public int getLineNumber() {
return lineNumber;
}
public int getEndOffset() { return getOffset() + getLength(); }
protected int type;
protected int lineNumber = 1;
protected IToken next = null;
/* (non-Javadoc)
* @see java.lang.Object#equals(java.lang.Object)
*/
public boolean equals(Object other) {
if( other == null ) return false;
if( !( other instanceof IToken ) )
return false;
if( ((IToken)other).getType() != getType() )
return false;
if( !(((IToken)other).getImage().equals( getImage() )))
return false;
return true;
}
/* (non-Javadoc)
* @see org.eclipse.cdt.core.parser.IToken#isKeyword()
*/
public boolean isKeywordOrOperator() {
switch( getType() )
{
case IToken.tCHAR:
case IToken.tFLOATINGPT:
case IToken.tIDENTIFIER:
case IToken.tINTEGER:
case IToken.tSTRING:
case IToken.tLSTRING:
case IToken.tLCHAR:
return false;
default:
return true;
}
}
public boolean looksLikeExpression()
{
switch( getType() )
{
case IToken.tINTEGER:
case IToken.t_false:
case IToken.t_true:
case IToken.tSTRING:
case IToken.tLSTRING:
case IToken.tFLOATINGPT:
case IToken.tCHAR:
case IToken.tAMPER:
case IToken.tDOT:
case IToken.tLPAREN:
case IToken.tMINUS:
case IToken.tSTAR:
case IToken.tPLUS:
case IToken.tNOT:
case IToken.tCOMPL:
return true;
default:
break;
}
return false;
}
public boolean isOperator()
{
switch( getType() )
{
case IToken.t_new:
case IToken.t_delete:
case IToken.tPLUS:
case IToken.tMINUS:
case IToken.tSTAR:
case IToken.tDIV:
case IToken.tXOR:
case IToken.tMOD:
case IToken.tAMPER:
case IToken.tBITOR:
case IToken.tCOMPL:
case IToken.tNOT:
case IToken.tASSIGN:
case IToken.tLT:
case IToken.tGT:
case IToken.tPLUSASSIGN:
case IToken.tMINUSASSIGN:
case IToken.tSTARASSIGN:
case IToken.tDIVASSIGN:
case IToken.tMODASSIGN:
case IToken.tBITORASSIGN:
case IToken.tAMPERASSIGN:
case IToken.tXORASSIGN:
case IToken.tSHIFTL:
case IToken.tSHIFTR:
case IToken.tSHIFTLASSIGN:
case IToken.tSHIFTRASSIGN:
case IToken.tEQUAL:
case IToken.tNOTEQUAL:
case IToken.tLTEQUAL:
case IToken.tGTEQUAL:
case IToken.tAND:
case IToken.tOR:
case IToken.tINCR:
case IToken.tDECR:
case IToken.tCOMMA:
case IToken.tARROW:
case IToken.tARROWSTAR:
return true;
default:
return false;
}
}
public boolean isPointer()
{
return (getType() == IToken.tAMPER || getType() == IToken.tSTAR);
}
public final IToken getNext() { return next; }
public void setNext(IToken t) { next = t; }
}

48
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedExpansionToken.java Normal file
View file

@ -0,0 +1,48 @@
/**********************************************************************
* Copyright (c) 2002-2004 IBM Canada and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Common Public License v0.5
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/cpl-v05.html
*
* Contributors:
* IBM Rational Software - Initial API and implementation
***********************************************************************/
package org.eclipse.cdt.internal.core.parser.token;
import org.eclipse.cdt.core.parser.IToken;
import org.eclipse.cdt.internal.core.parser.scanner.ContextStack;
import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
/**
* @author johnc
*/
public class ImagedExpansionToken extends ImagedToken implements IToken {
/**
* @param t
* @param contextStack
* @param i
*/
public ImagedExpansionToken(int t, ContextStack contextStack, String i) {
super(t, contextStack, i);
}
protected int length;
/* (non-Javadoc)
* @see org.eclipse.cdt.internal.core.parser.token.SimpleToken#setOffsetAndLength(org.eclipse.cdt.internal.core.parser.scanner.IScannerContext)
*/
protected void setOffsetAndLength(IScannerContext context) {
offset = context.getMacroOffset();
length = context.getMacroLength();
}
/* (non-Javadoc)
* @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getLength()
*/
public final int getLength() {
return length;
}
}

78
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedToken.java Normal file
View file

@ -0,0 +1,78 @@
/**********************************************************************
* Copyright (c) 2002-2004 IBM Canada and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Common Public License v0.5
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/cpl-v05.html
*
* Contributors:
* IBM Rational Software - Initial API and implementation
***********************************************************************/
package org.eclipse.cdt.internal.core.parser.token;
import org.eclipse.cdt.internal.core.parser.scanner.ContextStack;
import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
/**
* @author johnc
*/
public class ImagedToken extends SimpleToken {
protected String image = null;
/**
* @param t
* @param contextStack
* @param i
*/
public ImagedToken(int t, ContextStack contextStack, String i) {
super(t, contextStack);
setImage( i );
setOffsetAndLength(contextStack.getCurrentContext());
}
public ImagedToken( int t, String i) {
super( t );
setImage(i);
}
/* (non-Javadoc)
* @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getImage()
*/
public final String getImage() {
return image;
}
/* (non-Javadoc)
* @see org.eclipse.cdt.core.parser.IToken#setImage(java.lang.String)
*/
public void setImage(String i) {
image = i;
}
/**
* @param context
*/
protected void setOffsetAndLength(IScannerContext context) {
if( getImage() == null ) return;
offset = context.getOffset() - getImage().length() - context.undoStackSize();
if( getType() == tSTRING || getType() == tCHAR )
offset--;
else if( getType() == tLSTRING || getType() == tLCHAR )
offset -= 2;
}
public int getLength() {
switch( getType() )
{
case tSTRING:
case tCHAR:
return getImage().length() + 2; // 'c' is 3 characters, not 1
case tLSTRING:
case tLCHAR:
return getImage().length() + 3; // L"X" if 4 characters, not 1
default:
return getImage().length();
}
}
}

48
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleExpansionToken.java Normal file
View file

@ -0,0 +1,48 @@
/**********************************************************************
* Copyright (c) 2002-2004 IBM Canada and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Common Public License v0.5
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/cpl-v05.html
*
* Contributors:
* IBM Rational Software - Initial API and implementation
***********************************************************************/
package org.eclipse.cdt.internal.core.parser.token;
import org.eclipse.cdt.core.parser.IToken;
import org.eclipse.cdt.internal.core.parser.scanner.ContextStack;
import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
/**
* @author johnc
*/
public class SimpleExpansionToken extends SimpleToken implements IToken {
/**
* @param tokenType
* @param stack
*/
public SimpleExpansionToken(int tokenType, ContextStack stack) {
super( tokenType, stack );
}
protected int length;
/* (non-Javadoc)
* @see org.eclipse.cdt.internal.core.parser.token.SimpleToken#setOffsetAndLength(org.eclipse.cdt.internal.core.parser.scanner.IScannerContext)
*/
protected void setOffsetAndLength(IScannerContext context) {
offset = context.getMacroOffset();
length = context.getMacroLength();
}
/* (non-Javadoc)
* @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getLength()
*/
public final int getLength() {
return length;
}
}

231
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/Token.java → core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleToken.java
View file

@ -10,71 +10,45 @@
******************************************************************************/
package org.eclipse.cdt.internal.core.parser.token;
import org.eclipse.cdt.core.parser.Keywords;
import org.eclipse.cdt.core.parser.IScanner;
import org.eclipse.cdt.core.parser.IToken;
import org.eclipse.cdt.internal.core.parser.scanner.*;
import org.eclipse.cdt.core.parser.Keywords;
import org.eclipse.cdt.internal.core.parser.scanner.ContextStack;
import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
public class Token implements IToken {
public class SimpleToken extends AbstractToken implements IToken {
public Token(int t, String i, IScannerContext context, int lineNumber ) {
setType(t);
setImage(i);
filename = context.getFilename();
offset = context.getOffset() - getImage().length() - context.undoStackSize();
this.lineNumber = lineNumber;
macroOffset = context.getMacroOffset();
macroLength = context.getMacroLength();
if( type == tLSTRING || type == tSTRING || type == tCHAR ){
offset--;
}
}
public Token(int t, IScannerContext context, int lineNumber ) {
setType(t);
setImage(null);
filename = context.getFilename();
offset = context.getOffset() - getImage().length() - context.undoStackSize();
this.lineNumber = lineNumber;
macroOffset = context.getMacroOffset();
macroLength = context.getMacroLength();
if( type == tLSTRING || type == tSTRING || type == tCHAR ){
offset--;
}
public SimpleToken(int t, ContextStack contextStack ) {
super(t,contextStack.getCurrentLineNumber());
setOffsetAndLength(contextStack.getCurrentContext());
}
public Token(int t) {
setType(t);
setImage(null);
}
public Token(int t, String i) {
setType(t);
setImage(i);
}
public String toString()
public SimpleToken( int t )
{
return "Token=" + type + " \"" + getImage() + "\" " + filename + ":" + lineNumber + " offset=" + offset; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
super( t );
}
public int type;
public int getType() { return type; }
protected int offset;
protected String image;
// All the tokens generated by the macro expansion
// will have dimensions (offset and length) equal to the expanding symbol.
public int getOffset() {
return offset;
}
public int getLength() {
return getImage().length();
}
/**
* @param context
*/
protected void setOffsetAndLength(IScannerContext context) {
offset = context.getOffset() - getImage().length() - context.undoStackSize();
}
public String getImage() {
switch ( getType() ) {
case IToken.tIDENTIFIER :
case IToken.tINTEGER :
case IToken.tFLOATINGPT :
case IToken.tSTRING :
case IToken.tLSTRING :
case IToken.tCHAR :
case IToken.tLCHAR :
return image;
case IToken.tCOLONCOLON :
return "::" ; //$NON-NLS-1$
case IToken.tCOLON :
@ -336,164 +310,17 @@ public class Token implements IToken {
default :
// we should never get here!
return image;
// assert false : getType();
return ""; //$NON-NLS-1$
}
}
public String filename;
protected int offset;
protected int macroOffset = -1;
protected int macroLength = -1;
// All the tokens generated by the macro expansion
// will have dimensions (offset and length) equal to the expanding symbol.
public int getOffset() { return (macroOffset < 0) ? offset : macroOffset; }
public int getLength() { return (macroLength < 0) ? getImage().length() : macroLength; }
public int getEndOffset() { return getOffset() + getLength(); }
public int getDelta( IToken other )
{
return other.getOffset() + other.getLength() - getOffset();
}
private IToken next;
private int lineNumber;
public IToken getNext() { return next; }
public void setNext(IToken t) { next = t; }
public boolean looksLikeExpression()
{
switch( getType() )
{
case tINTEGER:
case t_false:
case t_true:
case tSTRING:
case tLSTRING:
case tFLOATINGPT:
case tCHAR:
case tAMPER:
case tDOT:
case tLPAREN:
case tMINUS:
case tSTAR:
case tPLUS:
case tNOT:
case tCOMPL:
return true;
default:
break;
}
return false;
}
public boolean isPointer()
{
return (getType() == tAMPER || getType() == tSTAR);
}
public boolean isOperator()
{
switch( getType() )
{
case IToken.t_new:
case IToken.t_delete:
case IToken.tPLUS:
case IToken.tMINUS:
case IToken.tSTAR:
case IToken.tDIV:
case IToken.tXOR:
case IToken.tMOD:
case IToken.tAMPER:
case IToken.tBITOR:
case IToken.tCOMPL:
case IToken.tNOT:
case IToken.tASSIGN:
case IToken.tLT:
case IToken.tGT:
case IToken.tPLUSASSIGN:
case IToken.tMINUSASSIGN:
case IToken.tSTARASSIGN:
case IToken.tDIVASSIGN:
case IToken.tMODASSIGN:
case IToken.tBITORASSIGN:
case IToken.tAMPERASSIGN:
case IToken.tXORASSIGN:
case IToken.tSHIFTL:
case IToken.tSHIFTR:
case IToken.tSHIFTLASSIGN:
case IToken.tSHIFTRASSIGN:
case IToken.tEQUAL:
case IToken.tNOTEQUAL:
case IToken.tLTEQUAL:
case IToken.tGTEQUAL:
case IToken.tAND:
case IToken.tOR:
case IToken.tINCR:
case IToken.tDECR:
case IToken.tCOMMA:
case IToken.tARROW:
case IToken.tARROWSTAR:
return true;
default:
return false;
}
}
/* (non-Javadoc)
* @see org.eclipse.cdt.core.parser.IToken#setImage()
*/
public void setImage( String i ) {
image = i;
}
/* (non-Javadoc)
* @see java.lang.Object#equals(java.lang.Object)
*/
public boolean equals(Object other) {
if( other == null ) return false;
if( !( other instanceof IToken ) )
return false;
if( !(((IToken)other).getImage().equals( getImage() )))
return false;
if( ((IToken)other).getType() != type )
return false;
return true;
}
/* (non-Javadoc)
* @see org.eclipse.cdt.core.parser.IToken#setType(int)
*/
public void setType(int i) {
type = i;
}
/* (non-Javadoc)
* @see org.eclipse.cdt.core.parser.IToken#getLineNumber()
*/
public int getLineNumber() {
return lineNumber;
}
/* (non-Javadoc)
* @see org.eclipse.cdt.core.parser.IToken#isKeyword()
*/
public boolean isKeywordOrOperator() {
if( type == IToken.tCHAR ) return false;
if( type == IToken.tFLOATINGPT ) return false;
if( type == IToken.tIDENTIFIER ) return false;
if( type == IToken.tINTEGER ) return false;
if( type == IToken.tSTRING ) return false;
if( type == IToken.tLSTRING ) return false;
if( type == IToken.tLCHAR ) return false;
return true;
// do nothing
}
}

47
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java Normal file
View file

@ -0,0 +1,47 @@
/**********************************************************************
* Copyright (c) 2002-2004 IBM Canada and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Common Public License v0.5
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/cpl-v05.html
*
* Contributors:
* IBM Rational Software - Initial API and implementation
***********************************************************************/
package org.eclipse.cdt.internal.core.parser.token;
import org.eclipse.cdt.core.parser.IToken;
import org.eclipse.cdt.internal.core.parser.scanner.IScannerData;
/**
* @author johnc
*/
public class TokenFactory {
public static IToken createToken( int tokenType, IScannerData scannerData )
{
if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 )
return new SimpleExpansionToken( tokenType, scannerData.getContextStack() );
return new SimpleToken( tokenType, scannerData.getContextStack() );
}
/**
* @param type
* @param image
* @param scannerData
* @return
*/
public static IToken createToken(int type, String image, IScannerData scannerData) {
if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 )
return new ImagedExpansionToken( type, scannerData.getContextStack(), image );
return new ImagedToken(type, scannerData.getContextStack(), image );
}
public static IToken createToken( int type, String image )
{
return new ImagedToken( type, image);
}
}