Add toCharArray() to ITokenDuple. Add getCharImage() to IToken.

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/CompletionParseTest.java

@@ -845,7 +845,7 @@ public class CompletionParseTest extends CompletionParseBaseTest {
 	public void testBug59134() throws Exception
 	{
 		String code = "int main(){ siz }"; //$NON-NLS-1$
-		IASTCompletionNode node = parse( code, code.indexOf(" siz") ); //$NON-NLS-1$
+		IASTCompletionNode node = parse( code, code.indexOf("siz") ); //$NON-NLS-1$
 		assertNotNull( node );
 		Iterator keywords = node.getKeywords();
 		boolean passed = false;

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/SelectionParseBaseTest.java

@@ -58,6 +58,7 @@ public class SelectionParseBaseTest extends CompleteParseBaseTest {
 		{
 			assertNotNull( result );
 			String filename = result.getFilename();
+			assertNotNull( filename );
 			assertTrue( !filename.equals( "")); //$NON-NLS-1$
 			return (IASTNode) result.getOffsetableNamedElement();
 		}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java

@@ -20,6 +20,7 @@ public interface IToken {
 	// getters
 	public int getType();
 	public String getImage();
+	public char [] getCharImage();
 	public int getOffset();
 	public int getLength();
 	public int getEndOffset();
@@ -29,7 +30,8 @@ public interface IToken {
 	public IToken getNext();
 	
 	// setters
-	public void setImage( String i ); 
+	public void setImage( String i );
+	public void setImage( char [] i );
 	public void setNext(IToken t);
 	public void setType(int i);	
 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/ITokenDuple.java

@@ -37,6 +37,7 @@ public interface ITokenDuple {
 	
 	public abstract Iterator iterator();
 	public abstract String toString();
+	public char [] toCharArray();
 	public abstract boolean isIdentifier();
 	public abstract int length(); 
 	

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/MacroExpansionToken.java

@@ -131,4 +131,20 @@ public class MacroExpansionToken implements IToken {
 		return false;
 	}
 
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.IToken#getCharImage()
+	 */
+	public char[] getCharImage() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.IToken#setImage(char[])
+	 */
+	public void setImage(char[] i) {
+		// TODO Auto-generated method stub
+		
+	}
+
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java

@@ -326,7 +326,7 @@ public class Scanner2 implements IScanner, IScannerData {
 				int tokenType = IToken.tSTRING; 
 				if( lastToken.getType() == IToken.tLSTRING || nextToken.getType() == IToken.tLSTRING )
 					tokenType = IToken.tLSTRING;
-				lastToken = new ImagedToken(tokenType, lastToken.getImage() + nextToken.getImage(), nextToken.getEndOffset()); //TODO Fix this
+				lastToken = new ImagedToken(tokenType, (lastToken.getImage() + nextToken.getImage()).toCharArray(), nextToken.getEndOffset()); //TODO Fix this
 				if (oldToken != null)
 					oldToken.setNext(lastToken);
 				nextToken = fetchToken();
@@ -747,7 +747,7 @@ public class Scanner2 implements IScanner, IScannerData {
 		int tokenType = keywords.get(buffer, start, len);
 		char [] result = removedEscapedNewline( CharArrayUtils.extract( buffer, start, len ) );
 		if (tokenType == keywords.undefined)
-			return new ImagedToken(IToken.tIDENTIFIER, new String( result), bufferPos[bufferStackPos]+ 1);
+			return new ImagedToken(IToken.tIDENTIFIER, result, bufferPos[bufferStackPos]+ 1);
 		return new SimpleToken(tokenType, start + len);
 	}
 	
@@ -782,7 +782,7 @@ public class Scanner2 implements IScanner, IScannerData {
 		// We should really throw an exception if we didn't get the terminating
 		// quote before the end of buffer
 		
-		return new ImagedToken(tokenType, new String(buffer, stringStart, stringLen), stringStart + stringLen+ 1);
+		return new ImagedToken(tokenType, CharArrayUtils.extract(buffer, stringStart, stringLen), stringStart + stringLen+ 1);
 	}
 
 	private IToken scanCharLiteral(boolean b) {
@@ -799,7 +799,7 @@ public class Scanner2 implements IScanner, IScannerData {
 		}
 
 		if (start >= limit) {
-			return new ImagedToken(tokenType, new String(emptyCharArray), start);
+			return new ImagedToken(tokenType, emptyCharArray, start);
 		}
 
 		
@@ -822,7 +822,7 @@ public class Scanner2 implements IScanner, IScannerData {
 			? CharArrayUtils.extract(buffer, start, length)
 			: emptyCharArray;
 
-		return new ImagedToken(tokenType, new String(image), start + length+ 1 );
+		return new ImagedToken(tokenType, image, start + length+ 1 );
 	}
 	
 	private IToken scanNumber() {
@@ -999,7 +999,7 @@ public class Scanner2 implements IScanner, IScannerData {
 		--bufferPos[bufferStackPos];
 		
 		return new ImagedToken(isFloat ? IToken.tFLOATINGPT : IToken.tINTEGER,
-				new String(buffer, start,
+				CharArrayUtils.extract(buffer, start,
 						bufferPos[bufferStackPos] - start + 1), bufferPos[bufferStackPos]+ 1);
 	}
 	

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/BasicTokenDuple.java

@@ -494,5 +494,12 @@ public class BasicTokenDuple implements ITokenDuple {
 	public void acceptElement(ISourceElementRequestor requestor, IReferenceManager manager) {
 	}
 
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.ITokenDuple#toCharArray()
+	 */
+	public char[] toCharArray() {
+		return toString().toCharArray(); //TODO fix me!
+	}
+
 	
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedExpansionToken.java

@@ -25,7 +25,7 @@ public class ImagedExpansionToken extends ImagedToken implements IToken {
 	 * @param contextStack
 	 * @param i
 	 */
-	public ImagedExpansionToken(int t, ContextStack contextStack, String i) {
+	public ImagedExpansionToken(int t, ContextStack contextStack, char[] i) {
 		super(t, contextStack, i);
 	}
 	
@@ -34,7 +34,7 @@ public class ImagedExpansionToken extends ImagedToken implements IToken {
 	 * @param contextStack
 	 * @param i
 	 */
-	public ImagedExpansionToken(int t, String i, int macroOffset, int macroLength ) {
+	public ImagedExpansionToken(int t, char[] i, int macroOffset, int macroLength ) {
 		super(t, i, macroOffset );
 		setOffsetAndLength( macroOffset, macroLength );
 	}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedToken.java

@@ -18,20 +18,20 @@ import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
  */
 public class ImagedToken extends SimpleToken {
 
-	protected String image = null;
+	protected char [] image = null;
 	
 	/**
 	 * @param t
 	 * @param contextStack
 	 * @param i
 	 */
-	public ImagedToken(int t, ContextStack contextStack, String i) {
+	public ImagedToken(int t, ContextStack contextStack, char[] i) {
 		super(t, contextStack);
 		setImage( i );
 		setOffsetAndLength(contextStack.getCurrentContext());
 	}
 	
-	public ImagedToken( int t, String i, int endOffset) {
+	public ImagedToken( int t, char[] i, int endOffset) {
 		super( t, 0 );
 		setImage(i);
 		setOffsetAndLength( endOffset );
@@ -42,13 +42,19 @@ public class ImagedToken extends SimpleToken {
 	 * @see org.eclipse.cdt.internal.core.parser.token.AbstractToken#getImage()
 	 */
 	public final String getImage() {
-		return image;
+		if( image == null ) return null;
+		return new String( image );
 	}
 	/* (non-Javadoc)
 	 * @see org.eclipse.cdt.core.parser.IToken#setImage(java.lang.String)
 	 */
 	public void setImage(String i) {
-		image = i;
+		image = i.toCharArray();
+	}
+	
+	public void setImage( char [] image )
+	{
+		this.image = image;
 	}
 	
 	/**

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleToken.java

@@ -329,4 +329,25 @@ public class SimpleToken extends AbstractToken implements IToken {
 		// do nothing
 	}
 
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.IToken#getCharImage()
+	 */
+	public char[] getCharImage() {
+		return getImage().toCharArray(); //TODO - fix me!
+	}
+
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.IToken#setImage(char[])
+	 */
+	public void setImage(char[] i) {
+		// do nothing
+	}
+
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.ITokenDuple#toCharArray()
+	 */
+	public char[] toCharArray() {
+		return getCharImage();
+	}
+
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java

@@ -40,14 +40,14 @@ public class TokenFactory {
 	 */
 	public static IToken createUniquelyImagedToken(int type, String image, IScannerData scannerData) {
 		if( scannerData.getContextStack().getCurrentContext().getKind() == IScannerContext.ContextKind.MACROEXPANSION )
-			return new ImagedExpansionToken( type, scannerData.getContextStack(), image );
+			return new ImagedExpansionToken( type, scannerData.getContextStack(), image.toCharArray() );
 
-		return new ImagedToken(type, scannerData.getContextStack(), image );
+		return new ImagedToken(type, scannerData.getContextStack(), image.toCharArray() );
 	}
 	
 	public static IToken createStandAloneToken( int type, String image )
 	{
-		return new ImagedToken( type, image, 0);
+		return new ImagedToken( type, image.toCharArray(), 0);
 	}
 
 	public static ITokenDuple createTokenDuple( IToken first, IToken last )