Migrate AsmModelBuilder to new Lexer

core/org.eclipse.cdt.core.tests/model/org/eclipse/cdt/core/model/tests/AsmModelBuilderTest.java

@@ -16,6 +16,7 @@ import org.eclipse.cdt.core.dom.IPDOMManager;
 import org.eclipse.cdt.core.model.IAsmLabel;
 import org.eclipse.cdt.core.model.ICElement;
 import org.eclipse.cdt.core.model.ICProject;
+import org.eclipse.cdt.core.model.IInclude;
 import org.eclipse.cdt.core.model.IParent;
 import org.eclipse.cdt.core.model.ISourceRange;
 import org.eclipse.cdt.core.model.ISourceReference;
@@ -58,38 +59,44 @@ public class AsmModelBuilderTest extends BaseTestCase {
 	
 	public void testAsmModelElements() throws Exception {
 		ICElement[] children= fTU.getChildren();
-		assertEquals(7, children.length);
+		assertEquals(8, children.length);
 		
-		assertEquals(ICElement.C_INCLUDE, children[0].getElementType());
+		int idx= 0;
-		assertEquals("include.h", children[0].getElementName());
+		assertEquals(ICElement.C_INCLUDE, children[idx].getElementType());
+		assertTrue(((IInclude)children[idx]).isStandard());
+		assertEquals("include1.h", children[idx++].getElementName());
+		assertEquals(ICElement.C_INCLUDE, children[idx].getElementType());
+		assertFalse(((IInclude)children[idx]).isStandard());
+		assertEquals("include2.h", children[idx++].getElementName());
 		
-		assertEquals(ICElement.ASM_LABEL, children[1].getElementType());
-		assertEquals("nonGlobalLabel", children[1].getElementName());
-		assertFalse(((IAsmLabel)children[1]).isGlobal());
-		assertEquals(0, ((IParent)children[1]).getChildren().length);
 		
-		assertEquals(ICElement.ASM_LABEL, children[2].getElementType());
+		assertEquals(ICElement.ASM_LABEL, children[idx].getElementType());
-		assertEquals("globalLabel1", children[2].getElementName());
+		assertEquals("nonGlobalLabel", children[idx].getElementName());
-		assertTrue(((IAsmLabel)children[2]).isGlobal());
+		assertFalse(((IAsmLabel)children[idx]).isGlobal());
-		assertEquals(2, ((IParent)children[2]).getChildren().length);
+		assertEquals(0, ((IParent)children[idx++]).getChildren().length);
 		
-		assertEquals(ICElement.C_MACRO, children[3].getElementType());
+		assertEquals(ICElement.ASM_LABEL, children[idx].getElementType());
-		assertEquals("MACRO", children[3].getElementName());
+		assertEquals("globalLabel1", children[idx].getElementName());
+		assertTrue(((IAsmLabel)children[idx]).isGlobal());
+		assertEquals(2, ((IParent)children[idx++]).getChildren().length);
 		
-		assertEquals(ICElement.ASM_LABEL, children[4].getElementType());
+		assertEquals(ICElement.C_MACRO, children[idx].getElementType());
-		assertEquals("globalLabel2", children[4].getElementName());
+		assertEquals("MACRO", children[idx++].getElementName());
-		assertTrue(((IAsmLabel)children[4]).isGlobal());
-		assertEquals(1, ((IParent)children[4]).getChildren().length);
 		
-		assertEquals(ICElement.ASM_LABEL, children[5].getElementType());
+		assertEquals(ICElement.ASM_LABEL, children[idx].getElementType());
-		assertEquals("globalLabel3", children[5].getElementName());
+		assertEquals("globalLabel2", children[idx].getElementName());
-		assertTrue(((IAsmLabel)children[5]).isGlobal());
+		assertTrue(((IAsmLabel)children[idx]).isGlobal());
-		assertEquals(1, ((IParent)children[5]).getChildren().length);
+		assertEquals(1, ((IParent)children[idx++]).getChildren().length);
 		
-		assertEquals(ICElement.ASM_LABEL, children[6].getElementType());
+		assertEquals(ICElement.ASM_LABEL, children[idx].getElementType());
-		assertEquals("alloca", children[6].getElementName());
+		assertEquals("globalLabel3", children[idx].getElementName());
-		assertTrue(((IAsmLabel)children[6]).isGlobal());
+		assertTrue(((IAsmLabel)children[idx]).isGlobal());
-		assertEquals(0, ((IParent)children[6]).getChildren().length);
+		assertEquals(1, ((IParent)children[idx++]).getChildren().length);
+		
+		assertEquals(ICElement.ASM_LABEL, children[idx].getElementType());
+		assertEquals("alloca", children[idx].getElementName());
+		assertTrue(((IAsmLabel)children[idx]).isGlobal());
+		assertEquals(0, ((IParent)children[idx++]).getChildren().length);
 	}
 
 	public void testAsmLabelRanges() throws Exception {

core/org.eclipse.cdt.core.tests/resources/asmTests/AsmTest.S

@@ -2,7 +2,8 @@
  * AsmTest.S
  */
 
-#include <include.h>
+#include <include1.h>
+#include "include2.h"
 
 	.globl globalLabel1
 	.global globalLabel2, globalLabel3
@@ -47,6 +48,7 @@ globalLabel3:
 otherLabel3:
 	movb	$0x81, %ah
 	jmp	globalLabel3
+	nop
 /* end */
 
 	.global alloca

core/org.eclipse.cdt.core/model/org/eclipse/cdt/internal/core/model/AsmModelBuilder.java

@@ -12,14 +12,19 @@
 package org.eclipse.cdt.internal.core.model;
 
 import java.util.HashMap;
+import java.util.Map;
 
 import org.eclipse.cdt.core.model.AssemblyLanguage;
 import org.eclipse.cdt.core.model.CModelException;
 import org.eclipse.cdt.core.model.IContributedModelBuilder;
 import org.eclipse.cdt.core.model.ITranslationUnit;
 import org.eclipse.cdt.core.parser.CodeReader;
-import org.eclipse.cdt.internal.formatter.scanner.Scanner;
+import org.eclipse.cdt.core.parser.IToken;
-import org.eclipse.cdt.internal.formatter.scanner.Token;
+import org.eclipse.cdt.core.parser.OffsetLimitReachedException;
+import org.eclipse.cdt.internal.core.parser.scanner.ILexerLog;
+import org.eclipse.cdt.internal.core.parser.scanner.Lexer;
+import org.eclipse.cdt.internal.core.parser.scanner.Token;
+import org.eclipse.cdt.internal.core.parser.scanner.Lexer.LexerOptions;
 
 /**
  * A simple model builder for assembly translation units.
@@ -31,31 +36,44 @@ import org.eclipse.cdt.internal.formatter.scanner.Token;
  */
 public class AsmModelBuilder implements IContributedModelBuilder {
 
+	private static final class AsmDirective {
+		private static final AsmDirective GLOBAL= new AsmDirective();
+		private static final AsmDirective DATA= new AsmDirective();
+	}
+
+	private static final class LexerLog implements ILexerLog {
+		public void handleComment(boolean isBlockComment, int offset, int endOffset) {
+		}
+		public void handleProblem(int problemID, char[] info, int offset, int endOffset) {
+		}
+	}
+
 	private static final class Counter {
 		int fCount;
 	}
 
-	private final static class Tokenizer extends Scanner {
+	private static final Map fgDirectives;
-		public Tokenizer(char[] source) {
-			setSource(source);
-		}
 	
-		public Token nextToken() {
+	static {
-			Token t= super.nextToken();
+		fgDirectives= new HashMap();
-			while (t != null && (t.isWhiteSpace())) {
+		fgDirectives.put("globl", AsmDirective.GLOBAL); //$NON-NLS-1$
-				t= super.nextToken();
+		fgDirectives.put("global", AsmDirective.GLOBAL); //$NON-NLS-1$
-			}
+		fgDirectives.put("ascii", AsmDirective.DATA); //$NON-NLS-1$
-			return t;
+		fgDirectives.put("asciz", AsmDirective.DATA); //$NON-NLS-1$
-		}
+		fgDirectives.put("byte", AsmDirective.DATA); //$NON-NLS-1$
+		fgDirectives.put("long", AsmDirective.DATA); //$NON-NLS-1$
+		fgDirectives.put("word", AsmDirective.DATA); //$NON-NLS-1$
 	}
 	
 	private TranslationUnit fTranslationUnit;
 	private char fLineSeparatorChar;
-	private HashMap fGlobals;
+	private Map fGlobals;
-	private HashMap fLabels;
+	private Map fLabels;
+	
 	private int fLastLabelEndOffset;
 	private AsmLabel fLastGlobalLabel;
 	private SourceManipulation fLastLabel;
+	private Lexer fLexer;
 
 	/**
 	 * Creates a model builder for the given assembly translation unit.
@@ -87,8 +105,10 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 		buildModel(reader.buffer);
 		// not sure whether this is necessary or not
 		fTranslationUnit.setIsStructureKnown(true);
+		// cleanup
 		fGlobals= null;
 		fLabels= null;
+		fLexer= null;
 	}
 
 	/**
@@ -96,6 +116,7 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 	 * 
 	 * @param source
 	 * @throws CModelException
+	 * @throws OffsetLimitReachedException 
 	 */
 	private void buildModel(final char[] source) throws CModelException {
 		fGlobals= new HashMap();
@@ -104,10 +125,8 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 		fLastGlobalLabel= null;
 		fLastLabelEndOffset= 0;
 
-		// TLETODO use new Lexer?
+		final LexerOptions lexerOptions= new LexerOptions();
-		Scanner scanner= new Scanner();
+		fLexer= new Lexer(source, lexerOptions, new LexerLog(), null);
-		scanner.setSplitPreprocessor(true);
-		scanner.setSource(source);
 		
 		// if true the next token is the first on a (logical) line
 		boolean firstTokenOnLine= true;
@@ -116,56 +135,39 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 		// inside instruction
 		boolean inInstruction= false;
 
-		Token token= scanner.nextToken();
+		IToken token= nextToken();
 		while (token != null) {
-			int type= adaptTokenType(token);
+			switch (token.getType()) {
-			switch (type) {
+			case IToken.tPOUND:
-			case Token.tPREPROCESSOR_INCLUDE:
+				if (fLexer.currentTokenIsFirstOnLine()) {
-				parseInclude(fTranslationUnit, token);
+					parsePPDirective(fTranslationUnit);
-				break;
-			case Token.tPREPROCESSOR_DEFINE:
-				parseDefine(fTranslationUnit, token);
-				break;
-			case Token.tFLOATINGPT:
-				if (token.getText().startsWith(".")) { //$NON-NLS-1$
-					// move scanner behind '.'
-					scanner.setCurrentPosition(scanner.getCurrentTokenStartPosition() + 1);
-					// fallthrough
-				} else {
-					break;
 				}
-			case Token.tDOT:
+				break;
+			case IToken.tDOT:
 				// assembly directive?
 				firstTokenOnLine= false;
 				if (expectInstruction) {
 					expectInstruction= false;
-					int pos= scanner.getCurrentPosition();
+					token= nextToken();
-					token= scanner.nextToken();
+					if (token != null && token.getType() == IToken.tIDENTIFIER) {
-					if (token != null && adaptTokenType(token) == Token.tIDENTIFIER) {
+						String text= token.getImage();
-						String text= token.getText();
 						if (isGlobalDirective(text)) {
-							firstTokenOnLine= parseGlobalDirective(token, scanner);
+							firstTokenOnLine= parseGlobalDirective(token);
 						} else if (isDataDirective(text)) {
 							inInstruction= true;
-							fLastLabelEndOffset= scanner.getCurrentTokenEndPosition();
 						}
-					} else {
-						scanner.setCurrentPosition(pos);
 					}
 				}
 				break;
-			case Token.tIDENTIFIER:
+			case IToken.tIDENTIFIER:
 				// identifier may be a label or part of an instruction
 				if (firstTokenOnLine) {
-					int pos= scanner.getCurrentPosition();
+					// peek next char
-					int ch= scanner.getNextChar();
+					char nextChar= source[token.getEndOffset()];
-					if (ch == ':') {
+					if (nextChar == ':') {
-						parseLabel(fTranslationUnit, token);
+						createLabel(fTranslationUnit, token);
-						fLastLabelEndOffset= scanner.getCurrentPosition();
 						expectInstruction= true;
 					} else {
-						fLastLabelEndOffset= scanner.getCurrentTokenEndPosition();
-						scanner.setCurrentPosition(pos);
 						expectInstruction= false;
 						inInstruction= true;
 					}
@@ -173,18 +175,13 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 				} else if (expectInstruction){
 					expectInstruction= false;
 					inInstruction= true;
-					fLastLabelEndOffset= scanner.getCurrentTokenEndPosition();
-				} else if (inInstruction) {
-					fLastLabelEndOffset= scanner.getCurrentTokenEndPosition();
 				}
 				break;
-			case Token.tPREPROCESSOR:
+			case Lexer.tNEWLINE:
-			case Token.tWHITESPACE:
 				if (!firstTokenOnLine) {
-					int nlIndex= token.getText().indexOf('\n');
+					firstTokenOnLine= true;
-					firstTokenOnLine= nlIndex >= 0;
+					if (inInstruction) {
-					if (firstTokenOnLine && inInstruction) {
+						fLastLabelEndOffset= fLexer.currentToken().getEndOffset();
-						fLastLabelEndOffset= scanner.getCurrentTokenStartPosition() + nlIndex + 1;
 					}
 				}
 				break;
@@ -192,7 +189,7 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 				expectInstruction= false;
 				firstTokenOnLine= false;
 				if (fLineSeparatorChar != 0) {
-					if (token.getLength() == 1 && token.getText().charAt(0) == fLineSeparatorChar) {
+					if (token.getLength() == 1 && token.getCharImage()[0] == fLineSeparatorChar) {
 						firstTokenOnLine= true;
 					}
 				}
@@ -201,10 +198,10 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 				expectInstruction= true;
 				inInstruction= false;
 			}
-			token= scanner.nextToken();
+			token= nextToken();
 		}
 		if (!firstTokenOnLine && inInstruction) {
-			fLastLabelEndOffset= scanner.getCurrentTokenEndPosition();
+			fLastLabelEndOffset= fLexer.currentToken().getEndOffset();
 		}
 		if (fLastLabel != null) {
 			fixupLastLabel();
@@ -214,45 +211,36 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 		}
 	}
 	
-	/**
+	protected IToken nextToken() {
-	 * Adapt non-identifier tokens to identifier type if they look like valid asm words.
+		Token token;
-	 * 
+		try {
-	 * @param token
+			token= fLexer.nextToken();
-	 * @return the adapted token type
+			if (token.getType() == Lexer.tEND_OF_INPUT) {
-	 */
+				token = null;
-	private int adaptTokenType(Token token) {
-		int type= token.getType();
-		if (type != Token.tIDENTIFIER) {
-			if (Character.isUnicodeIdentifierStart(token.getText().charAt(0))) {
-				type= Token.tIDENTIFIER;
 			}
+		} catch (OffsetLimitReachedException exc) {
+			token= null;
 		}
-		return type;
+		return token;
 	}
 
-	private boolean isDataDirective(String directive) {
+	private boolean parseGlobalDirective(IToken token) {
-		return "byte".equals(directive) || "long".equals(directive) || "word".equals(directive) || "ascii".equals(directive); //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$ //$NON-NLS-4$
-	}
-
-	private boolean parseGlobalDirective(Token token, Scanner scanner) {
 		boolean eol= false;
 		do {
-			Token t= scanner.nextToken();
+			IToken t= nextToken();
 			if (t == null) {
 				break;
 			}
-			switch (adaptTokenType(t)) {
+			switch (t.getType()) {
-			case Token.tIDENTIFIER:
+			case IToken.tIDENTIFIER:
-				registerGlobalLabel(t.getText());
+				registerGlobalLabel(t.getImage());
 				break;
-			case Token.tWHITESPACE:
+			case Lexer.tNEWLINE:
-				if (token.getText().indexOf('\n') >= 0) {
 				eol= true;
-				}
 				break;
 			default:
 				if (fLineSeparatorChar != 0) {
-					if (token.getLength() == 1 && token.getText().charAt(0) == fLineSeparatorChar) {
+					if (token.getLength() == 1 && token.getCharImage()[0] == fLineSeparatorChar) {
 						eol= true;
 					}
 				}
@@ -261,11 +249,15 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 		return eol;
 	}
 
-	private boolean isGlobalDirective(String name) {
+	private static final boolean isDataDirective(String directive) {
-		return "globl".equals(name) || "global".equals(name); //$NON-NLS-1$ //$NON-NLS-2$
+		return fgDirectives.get(directive) == AsmDirective.DATA;
 	}
 
-	private void registerGlobalLabel(String labelName) {
+	private static final boolean isGlobalDirective(String directive) {
+		return fgDirectives.get(directive) == AsmDirective.GLOBAL;
+	}
+
+	protected void registerGlobalLabel(String labelName) {
 		Counter counter= (Counter) fGlobals.get(labelName);
 		if (counter == null) {
 			fGlobals.put(labelName, counter= new Counter());
@@ -281,7 +273,7 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 		return counter.fCount;
 	}
 
-	private int registerLabel(String labelName) {
+	protected int registerLabel(String labelName) {
 		Counter counter= (Counter) fLabels.get(labelName);
 		if (counter == null) {
 			fLabels.put(labelName, counter= new Counter());
@@ -289,8 +281,8 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 		return counter.fCount++;
 	}
 
-	private void parseLabel(CElement parent, Token token) throws CModelException {
+	protected void createLabel(CElement parent, IToken token) throws CModelException {
-		String labelName=  token.getText();
+		String labelName= token.getImage();
 		int index= getGlobalLabelIndex(labelName);
 		boolean global= index > 0;
 		if (!global) {
@@ -347,68 +339,93 @@ public class AsmModelBuilder implements IContributedModelBuilder {
 		}
 	}
 
-	private void parseDefine(CElement parent, Token token) throws CModelException {
+	private void parsePPDirective(CElement parent) throws CModelException {
-		final String tokenText= token.getText();
+		IToken token= nextToken();
-		Scanner defineScanner= new Tokenizer(tokenText.toCharArray());
+		if (token != null && token.getType() == IToken.tIDENTIFIER) {
-		defineScanner.setCurrentPosition(tokenText.indexOf('#') + 1);
+			final String image= token.getImage();
+			if (image.equals("define")) { //$NON-NLS-1$
+				try {
+					parsePPDefine(parent);
+				} catch (OffsetLimitReachedException exc) {
+				}
+				return;
+			} else if (image.equals("include")) { //$NON-NLS-1$
+				try {
+					parsePPInclude(parent);
+				} catch (OffsetLimitReachedException exc) {
+				}
+				return;
+			}
+		}
+		try {
+			skipToNewLine();
+		} catch (OffsetLimitReachedException exc) {
+		}
+	}
+
+	protected int skipToNewLine() throws OffsetLimitReachedException {
+		return fLexer.consumeLine(fLexer.currentToken().getEndOffset());
+	}
+
+	private void parsePPDefine(CElement parent) throws CModelException, OffsetLimitReachedException {
+		int startOffset= fLexer.currentToken().getOffset();
 		int nameStart= 0;
 		int nameEnd= 0;
 		String name= null;
-		Token t= defineScanner.nextToken();
+		IToken t= nextToken();
-		if (adaptTokenType(t) == Token.tIDENTIFIER && t.getText().equals("define")) { //$NON-NLS-1$
+		if (t.getType() == IToken.tIDENTIFIER) {
-			t= defineScanner.nextToken();
+			nameStart= fLexer.currentToken().getOffset();
-			if (adaptTokenType(t) == Token.tIDENTIFIER) {
+			nameEnd= fLexer.currentToken().getEndOffset();
-				nameStart= defineScanner.getCurrentTokenStartPosition();
+			name= t.getImage();
-				nameEnd= defineScanner.getCurrentTokenEndPosition() + 1;
-				name= t.getText();
-			}
 		}
 		if (name == null) {
 			return;
 		}
+		int endOffset= skipToNewLine();
 		Macro macro= new Macro(parent, name);
 		SourceManipulationInfo macroInfo= macro.getSourceManipulationInfo();
-		macroInfo.setIdPos(token.getOffset() + nameStart, nameEnd - nameStart);
+		macroInfo.setIdPos(nameStart, nameEnd - nameStart);
-		macroInfo.setPos(token.getOffset(), token.getLength());
+		macroInfo.setPos(startOffset, endOffset - startOffset);
 		parent.addChild(macro);
 	}
 
-	private void parseInclude(CElement parent, Token token) throws CModelException {
+	private void parsePPInclude(CElement parent) throws CModelException, OffsetLimitReachedException {
-		final String tokenText= token.getText();
+		int startOffset= fLexer.currentToken().getOffset();
-		Scanner includeScanner= new Tokenizer(tokenText.toCharArray());
-		includeScanner.setCurrentPosition(tokenText.indexOf('#') + 1);
 		int nameStart= 0;
-		int nameEnd= includeScanner.eofPosition + 1;
+		int nameEnd= 0;
 		String name= null;
 		boolean isStandard= false;
-		Token t= includeScanner.nextToken();
+		IToken t= nextToken();
-		if (adaptTokenType(t) == Token.tIDENTIFIER) {
+		switch (t.getType()) {
-			t= includeScanner.nextToken();
+		case IToken.tLT:
-			if (t.type == Token.tLT) {
+			nameStart= fLexer.currentToken().getOffset();
-				nameStart= includeScanner.getCurrentTokenStartPosition();
 			do {
-					t= includeScanner.nextToken();
+				t= nextToken();
-				} while (t != null && t.type != Token.tGT);
+			} while (t.getType() != IToken.tGT);
-				nameEnd= includeScanner.getCurrentTokenEndPosition() + 1;
+			nameEnd= fLexer.currentToken().getEndOffset();
-				name= tokenText.substring(nameStart + 1, nameEnd - 1);
+			name= new String(fLexer.getInputChars(nameStart + 1, nameEnd - 1));
-			} else if (t.type == Token.tSTRING) {
-				nameStart= includeScanner.getCurrentTokenStartPosition();
-				nameEnd= includeScanner.getCurrentTokenEndPosition() + 1;
-				name= t.getText().substring(1, t.getLength() - 1);
 			isStandard= true;
-			} else if (adaptTokenType(t) == Token.tIDENTIFIER) {
+			break;
-				nameStart= includeScanner.getCurrentTokenStartPosition();
+		case IToken.tSTRING:
-				nameEnd= includeScanner.getCurrentTokenEndPosition() + 1;
+			nameStart= fLexer.currentToken().getOffset();
-				name= t.getText();
+			nameEnd= fLexer.currentToken().getEndOffset();
-			}
+			name= t.getImage().substring(1, t.getLength() - 1);
+			break;
+		case IToken.tIDENTIFIER:
+			nameStart= fLexer.currentToken().getOffset();
+			nameEnd= fLexer.currentToken().getEndOffset();
+			name= t.getImage();
+			break;
+		default:
 		}
 		if (name == null) {
 			return;
 		}
+		int endOffset= skipToNewLine();
 		Include include= new Include(parent, name, isStandard);
 		SourceManipulationInfo includeInfo= include.getSourceManipulationInfo();
-		includeInfo.setIdPos(token.getOffset() + nameStart, nameEnd - nameStart);
+		includeInfo.setIdPos(nameStart, nameEnd - nameStart);
-		includeInfo.setPos(token.getOffset(), token.getLength());
+		includeInfo.setPos(startOffset, endOffset - startOffset);
 		parent.addChild(include);
 	}