From 1da410d13a1d6ea7ab36efec42eb8fecb40bc2b7 Mon Sep 17 00:00:00 2001 From: John Camelon Date: Tue, 19 Apr 2005 18:48:29 +0000 Subject: [PATCH] Further cleanup of Tokens for performance/memory. --- .../core/parser/scanner2/DOMScanner.java | 6 +- .../core/parser/token2/AbstractToken2.java | 340 ------------------ .../core/parser/token2/ImagedToken2.java | 8 +- .../core/parser/token2/SimpleToken2.java | 324 ++++++++++++++++- 4 files changed, 324 insertions(+), 354 deletions(-) delete mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/AbstractToken2.java diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/DOMScanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/DOMScanner.java index 5f294841754..d9c6425ce51 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/DOMScanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/DOMScanner.java @@ -284,14 +284,12 @@ public class DOMScanner extends BaseScanner { */ protected IToken newToken(int signal) { return new SimpleToken2(signal, - resolveOffset(bufferPos[bufferStackPos] + 1), - getLineNumber(bufferPos[bufferStackPos] + 1)); + resolveOffset(bufferPos[bufferStackPos] + 1)); } protected IToken newToken(int signal, char[] buffer) { IToken i = new ImagedToken2(signal, buffer, - resolveOffset(bufferPos[bufferStackPos] + 1), - getLineNumber(bufferPos[bufferStackPos] + 1)); + resolveOffset(bufferPos[bufferStackPos] + 1)); if (buffer != null && buffer.length == 0 && signal != IToken.tSTRING && signal != IToken.tLSTRING) bufferPos[bufferStackPos] += 1; // TODO - remove this hack at some diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/AbstractToken2.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/AbstractToken2.java deleted file mode 100644 index 75811a83169..00000000000 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/AbstractToken2.java +++ /dev/null @@ -1,340 +0,0 @@ -/********************************************************************** - * Copyright (c) 2004 IBM Corporation and others. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Common Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/cpl-v10.html - * - * Contributors: - * IBM - Initial API and implementation - **********************************************************************/ -package org.eclipse.cdt.internal.core.parser.token2; - -import java.util.Iterator; -import java.util.List; - -import org.eclipse.cdt.core.parser.ISourceElementRequestor; -import org.eclipse.cdt.core.parser.IToken; -import org.eclipse.cdt.core.parser.ITokenDuple; -import org.eclipse.cdt.core.parser.util.CharArrayUtils; - -public abstract class AbstractToken2 implements IToken, ITokenDuple { - - private static final char[] EMPTY_CHAR_ARRAY = {}; - - public AbstractToken2( int type, int lineNumber ) - { - setType( type ); - this.lineNumber = lineNumber; - } - - public String toString() { - return getImage(); - } - - public abstract String getImage(); - public abstract int getOffset(); - public abstract int getLength(); - - public int getType() { return type; } - - public void setType(int i) { - type = i; - } - - public int getLineNumber() { - return lineNumber; - } - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.IToken#getFilename() - */ - public char[] getFilename() { - return EMPTY_CHAR_ARRAY; - } - - public int getEndOffset() { return getOffset() + getLength(); } - - protected int type; - protected int lineNumber = 1; - protected IToken next = null; - - /* (non-Javadoc) - * @see java.lang.Object#equals(java.lang.Object) - */ - public boolean equals(Object other) { - if( other == null ) return false; - if( !( other instanceof IToken ) ) - return false; - if( ((IToken)other).getType() != getType() ) - return false; - if( !CharArrayUtils.equals( ((IToken)other).getCharImage(), getCharImage() ) ) - return false; - if( getOffset() != ((IToken)other).getOffset() ) - return false; - if( getEndOffset() != ((IToken)other).getEndOffset() ) - return false; - return true; - } - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.IToken#isKeyword() - */ - public boolean canBeAPrefix() { - switch( getType() ) - { - case tIDENTIFIER: - case tCOMPL: - return true; - default: - if( getType() >= t_and && getType() <= t_xor_eq ) return true; - if( getType() >= t__Bool && getType() <= t_restrict ) return true; - } - return false; - } - - public boolean looksLikeExpression() - { - switch( getType() ) - { - case IToken.tINTEGER: - case IToken.t_false: - case IToken.t_true: - case IToken.tSTRING: - case IToken.tLSTRING: - case IToken.tFLOATINGPT: - case IToken.tCHAR: - case IToken.tAMPER: - case IToken.tDOT: - case IToken.tLPAREN: - case IToken.tMINUS: - case IToken.tSTAR: - case IToken.tPLUS: - case IToken.tNOT: - case IToken.tCOMPL: - return true; - default: - break; - } - return false; - } - - public boolean isOperator() - { - switch( getType() ) - { - case IToken.t_new: - case IToken.t_delete: - case IToken.tPLUS: - case IToken.tMINUS: - case IToken.tSTAR: - case IToken.tDIV: - case IToken.tXOR: - case IToken.tMOD: - case IToken.tAMPER: - case IToken.tBITOR: - case IToken.tCOMPL: - case IToken.tNOT: - case IToken.tASSIGN: - case IToken.tLT: - case IToken.tGT: - case IToken.tPLUSASSIGN: - case IToken.tMINUSASSIGN: - case IToken.tSTARASSIGN: - case IToken.tDIVASSIGN: - case IToken.tMODASSIGN: - case IToken.tBITORASSIGN: - case IToken.tAMPERASSIGN: - case IToken.tXORASSIGN: - case IToken.tSHIFTL: - case IToken.tSHIFTR: - case IToken.tSHIFTLASSIGN: - case IToken.tSHIFTRASSIGN: - case IToken.tEQUAL: - case IToken.tNOTEQUAL: - case IToken.tLTEQUAL: - case IToken.tGTEQUAL: - case IToken.tAND: - case IToken.tOR: - case IToken.tINCR: - case IToken.tDECR: - case IToken.tCOMMA: - case IToken.tARROW: - case IToken.tARROWSTAR: - return true; - default: - return false; - } - } - - public boolean isPointer() - { - return (getType() == IToken.tAMPER || getType() == IToken.tSTAR); - } - - - - public final IToken getNext() { return next; } - public void setNext(IToken t) { next = t; } - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#contains(org.eclipse.cdt.core.parser.ITokenDuple) - */ - public boolean contains(ITokenDuple duple) { - return ( duple.getFirstToken() == duple.getLastToken() ) && ( duple.getFirstToken() == this ); - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#extractNameFromTemplateId() - */ - public char[] extractNameFromTemplateId(){ - return getCharImage(); - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#findLastTokenType(int) - */ - public int findLastTokenType(int t) { - if( getType() == t ) return 0; - return -1; - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#getFirstToken() - */ - public IToken getFirstToken() { - return this; - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#getLastSegment() - */ - public ITokenDuple getLastSegment() { - return this; - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#getLastToken() - */ - public IToken getLastToken() { - return this; - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#getLeadingSegments() - */ - public ITokenDuple getLeadingSegments() { - return null; - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#getSegmentCount() - */ - public int getSegmentCount() { - return 1; - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#getStartOffset() - */ - public int getStartOffset() { - return getOffset(); - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#getSubrange(int, int) - */ - public ITokenDuple getSubrange(int startIndex, int endIndex) { - if( startIndex == 0 && endIndex == 0 ) return this; - return null; - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#getTemplateIdArgLists() - */ - public List[] getTemplateIdArgLists() { - // TODO Auto-generated method stub - return null; - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#getToken(int) - */ - public IToken getToken(int index) { - if( index == 0 ) return this; - return null; - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#isIdentifier() - */ - public boolean isIdentifier() { - return ( getType() == IToken.tIDENTIFIER ); - } - - - private class SingleIterator implements Iterator - { - boolean hasNext = true; - /* (non-Javadoc) - * @see java.util.Iterator#remove() - */ - public void remove() { - throw new UnsupportedOperationException(); - } - - /* (non-Javadoc) - * @see java.util.Iterator#hasNext() - */ - public boolean hasNext() { - return hasNext; - } - - /* (non-Javadoc) - * @see java.util.Iterator#next() - */ - public Object next() { - hasNext = false; - return AbstractToken2.this; - } - - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#iterator() - */ - public Iterator iterator() { - return new SingleIterator(); - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#length() - */ - public int length() { - return 1; - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#syntaxOfName() - */ - public boolean syntaxOfName() { - return ( getType() == IToken.tIDENTIFIER ); - } - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#toQualifiedName() - */ - public String[] toQualifiedName() { - String [] qualifiedName = new String[1]; - qualifiedName[0] = getImage(); - return qualifiedName; - } - - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#freeReferences(org.eclipse.cdt.core.parser.ast.IReferenceManager) - */ - public void freeReferences() { - } - - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#acceptElement(org.eclipse.cdt.core.parser.ast.IReferenceManager) - */ - public void acceptElement(ISourceElementRequestor requestor) { - } - - /* (non-Javadoc) - * @see org.eclipse.cdt.core.parser.ITokenDuple#getSegmentIterator() - */ - public ITokenDuple[] getSegments() { - ITokenDuple [] r = new ITokenDuple[0]; - r[0] = this; - return r; - } - -} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/ImagedToken2.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/ImagedToken2.java index 119f4c2b4a4..3b60ff20616 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/ImagedToken2.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/ImagedToken2.java @@ -14,12 +14,12 @@ import org.eclipse.cdt.core.parser.IToken; public class ImagedToken2 extends SimpleToken2 { - protected char [] image = null; + private char [] image = null; - public ImagedToken2( int t, char[] i, int endOffset, int l ) { - super( t, 0, l ); + public ImagedToken2( int t, char[] i, int endOffset ) { + super( t, 0 ); setImage(i); - setOffsetAndLength( endOffset ); + setOffsetByLength( endOffset ); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/SimpleToken2.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/SimpleToken2.java index d2f6fd7a1cf..063306f70cb 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/SimpleToken2.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token2/SimpleToken2.java @@ -10,19 +10,331 @@ **********************************************************************/ package org.eclipse.cdt.internal.core.parser.token2; +import java.util.Iterator; +import java.util.List; + import org.eclipse.cdt.core.parser.IScanner; +import org.eclipse.cdt.core.parser.ISourceElementRequestor; import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.ITokenDuple; import org.eclipse.cdt.core.parser.Keywords; +import org.eclipse.cdt.core.parser.util.CharArrayUtils; -public class SimpleToken2 extends AbstractToken2 { +public class SimpleToken2 implements IToken, ITokenDuple { - public SimpleToken2( int t, int endOffset, int line ) + private static final char[] EMPTY_CHAR_ARRAY = {}; + + public SimpleToken2( int type, int endOffset ) { - super( t, line ); - setOffsetAndLength( endOffset ); + setType( type ); + setOffsetByLength( endOffset ); + } + + public String toString() { + return getImage(); + } + + public int getType() { return type; } + + public void setType(int i) { + type = i; } - protected int offset; + public int getLineNumber() { + return 0; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#getFilename() + */ + public char[] getFilename() { + return EMPTY_CHAR_ARRAY; + } + + public int getEndOffset() { return getOffset() + getLength(); } + + private int type; + private IToken next = null; + private int offset; + + /* (non-Javadoc) + * @see java.lang.Object#equals(java.lang.Object) + */ + public boolean equals(Object other) { + if( other == null ) return false; + if( !( other instanceof IToken ) ) + return false; + if( ((IToken)other).getType() != getType() ) + return false; + if( !CharArrayUtils.equals( ((IToken)other).getCharImage(), getCharImage() ) ) + return false; + if( getOffset() != ((IToken)other).getOffset() ) + return false; + if( getEndOffset() != ((IToken)other).getEndOffset() ) + return false; + return true; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#isKeyword() + */ + public boolean canBeAPrefix() { + switch( getType() ) + { + case tIDENTIFIER: + case tCOMPL: + return true; + default: + if( getType() >= t_and && getType() <= t_xor_eq ) return true; + if( getType() >= t__Bool && getType() <= t_restrict ) return true; + } + return false; + } + + public boolean looksLikeExpression() + { + switch( getType() ) + { + case IToken.tINTEGER: + case IToken.t_false: + case IToken.t_true: + case IToken.tSTRING: + case IToken.tLSTRING: + case IToken.tFLOATINGPT: + case IToken.tCHAR: + case IToken.tAMPER: + case IToken.tDOT: + case IToken.tLPAREN: + case IToken.tMINUS: + case IToken.tSTAR: + case IToken.tPLUS: + case IToken.tNOT: + case IToken.tCOMPL: + return true; + default: + break; + } + return false; + } + + public boolean isOperator() + { + switch( getType() ) + { + case IToken.t_new: + case IToken.t_delete: + case IToken.tPLUS: + case IToken.tMINUS: + case IToken.tSTAR: + case IToken.tDIV: + case IToken.tXOR: + case IToken.tMOD: + case IToken.tAMPER: + case IToken.tBITOR: + case IToken.tCOMPL: + case IToken.tNOT: + case IToken.tASSIGN: + case IToken.tLT: + case IToken.tGT: + case IToken.tPLUSASSIGN: + case IToken.tMINUSASSIGN: + case IToken.tSTARASSIGN: + case IToken.tDIVASSIGN: + case IToken.tMODASSIGN: + case IToken.tBITORASSIGN: + case IToken.tAMPERASSIGN: + case IToken.tXORASSIGN: + case IToken.tSHIFTL: + case IToken.tSHIFTR: + case IToken.tSHIFTLASSIGN: + case IToken.tSHIFTRASSIGN: + case IToken.tEQUAL: + case IToken.tNOTEQUAL: + case IToken.tLTEQUAL: + case IToken.tGTEQUAL: + case IToken.tAND: + case IToken.tOR: + case IToken.tINCR: + case IToken.tDECR: + case IToken.tCOMMA: + case IToken.tARROW: + case IToken.tARROWSTAR: + return true; + default: + return false; + } + } + + public boolean isPointer() + { + return (getType() == IToken.tAMPER || getType() == IToken.tSTAR); + } + + + + public final IToken getNext() { return next; } + public void setNext(IToken t) { next = t; } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#contains(org.eclipse.cdt.core.parser.ITokenDuple) + */ + public boolean contains(ITokenDuple duple) { + return ( duple.getFirstToken() == duple.getLastToken() ) && ( duple.getFirstToken() == this ); + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#extractNameFromTemplateId() + */ + public char[] extractNameFromTemplateId(){ + return getCharImage(); + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#findLastTokenType(int) + */ + public int findLastTokenType(int t) { + if( getType() == t ) return 0; + return -1; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#getFirstToken() + */ + public IToken getFirstToken() { + return this; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#getLastSegment() + */ + public ITokenDuple getLastSegment() { + return this; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#getLastToken() + */ + public IToken getLastToken() { + return this; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#getLeadingSegments() + */ + public ITokenDuple getLeadingSegments() { + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#getSegmentCount() + */ + public int getSegmentCount() { + return 1; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#getStartOffset() + */ + public int getStartOffset() { + return getOffset(); + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#getSubrange(int, int) + */ + public ITokenDuple getSubrange(int startIndex, int endIndex) { + if( startIndex == 0 && endIndex == 0 ) return this; + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#getTemplateIdArgLists() + */ + public List[] getTemplateIdArgLists() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#getToken(int) + */ + public IToken getToken(int index) { + if( index == 0 ) return this; + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#isIdentifier() + */ + public boolean isIdentifier() { + return ( getType() == IToken.tIDENTIFIER ); + } + + + private class SingleIterator implements Iterator + { + boolean hasNext = true; + /* (non-Javadoc) + * @see java.util.Iterator#remove() + */ + public void remove() { + throw new UnsupportedOperationException(); + } + + /* (non-Javadoc) + * @see java.util.Iterator#hasNext() + */ + public boolean hasNext() { + return hasNext; + } + + /* (non-Javadoc) + * @see java.util.Iterator#next() + */ + public Object next() { + hasNext = false; + return SimpleToken2.this; + } + + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#iterator() + */ + public Iterator iterator() { + return new SingleIterator(); + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#length() + */ + public int length() { + return 1; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#syntaxOfName() + */ + public boolean syntaxOfName() { + return ( getType() == IToken.tIDENTIFIER ); + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#toQualifiedName() + */ + public String[] toQualifiedName() { + String [] qualifiedName = new String[1]; + qualifiedName[0] = getImage(); + return qualifiedName; + } + + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#freeReferences(org.eclipse.cdt.core.parser.ast.IReferenceManager) + */ + public void freeReferences() { + } + + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#acceptElement(org.eclipse.cdt.core.parser.ast.IReferenceManager) + */ + public void acceptElement(ISourceElementRequestor requestor) { + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#getSegmentIterator() + */ + public ITokenDuple[] getSegments() { + ITokenDuple [] r = new ITokenDuple[0]; + r[0] = this; + return r; + } + // All the tokens generated by the macro expansion // will have dimensions (offset and length) equal to the expanding symbol. @@ -34,7 +346,7 @@ public class SimpleToken2 extends AbstractToken2 { return getCharImage().length; } - protected void setOffsetAndLength( int endOffset ) + protected void setOffsetByLength( int endOffset ) { this.offset = endOffset - getLength(); }