diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/LexerTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/LexerTests.java index 51da1384302..790bd86b4cf 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/LexerTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/LexerTests.java @@ -44,6 +44,7 @@ public class LexerTests extends BaseTestCase { private void init(String input) throws Exception { fLog.clear(); fLexer= new Lexer(input.toCharArray(), new LexerOptions(), fLog); + fLog.setInput(input); fLexer.nextToken(); fLastEndOffset= 0; } @@ -68,7 +69,7 @@ public class LexerTests extends BaseTestCase { assertEquals(fLastEndOffset, t.getOffset()); fLastEndOffset= t.getEndOffset(); if (image != null) { - assertEquals(image, new String(t.getTokenImage())); + assertEquals(image, new String(t.getCharImage())); } fLexer.nextToken(); } @@ -445,7 +446,7 @@ public class LexerTests extends BaseTestCase { init(instertLineSplices(input, splices)); for (int i = 0; i < tokens.length; i++) { Token token= fLexer.currentToken(); - buf.append(token.getTokenImage()); + buf.append(token.getCharImage()); token(tokens[i]); } eof(); diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/TestLexerLog.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/TestLexerLog.java index 6dd59a25e42..8581196dbbf 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/TestLexerLog.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/TestLexerLog.java @@ -18,13 +18,18 @@ public class TestLexerLog implements ILexerLog { private ArrayList fComments= new ArrayList(); private ArrayList fProblems= new ArrayList(); + private String fInput; - public void handleComment(boolean isBlockComment, char[] source, int offset, int endOffset) { - fComments.add(new String(source, offset, endOffset-offset)); + public void setInput(String input) { + fInput= input; } - public void handleProblem(int problemID, char[] source, int offset, int endOffset) { - fProblems.add(createString(problemID, new String(source, offset, endOffset-offset))); + public void handleComment(boolean isBlockComment, int offset, int endOffset) { + fComments.add(fInput.substring(offset, endOffset)); + } + + public void handleProblem(int problemID, char[] arg, int offset, int endOffset) { + fProblems.add(createString(problemID, new String(arg))); } public String createString(int problemID, String image) { diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ASTPreprocessorName.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ASTPreprocessorName.java new file mode 100644 index 00000000000..e98e1e2c732 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ASTPreprocessorName.java @@ -0,0 +1,120 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import org.eclipse.cdt.core.dom.ILinkage; +import org.eclipse.cdt.core.dom.ast.ASTNodeProperty; +import org.eclipse.cdt.core.dom.ast.IASTCompletionContext; +import org.eclipse.cdt.core.dom.ast.IASTFileLocation; +import org.eclipse.cdt.core.dom.ast.IASTName; +import org.eclipse.cdt.core.dom.ast.IASTNode; +import org.eclipse.cdt.core.dom.ast.IASTNodeLocation; +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit; +import org.eclipse.cdt.core.dom.ast.IBinding; +import org.eclipse.cdt.internal.core.dom.Linkage; + +/** + * Models IASTNames as needed for the preprocessor statements and macro expansions. + * @since 5.0 + */ +class ASTPreprocessorName extends ASTPreprocessorNode implements IASTName { + private final char[] fName; + private final IBinding fBinding; + public ASTPreprocessorName(IASTNode parent, ASTNodeProperty property, int startNumber, int endNumber, char[] name, IBinding binding) { + super(parent, property, startNumber, endNumber); + fName= name; + fBinding= binding; + } + + public IBinding resolveBinding() { + return fBinding; + } + public IBinding getBinding() { + return fBinding; + } + public ILinkage getLinkage() { + return Linkage.NO_LINKAGE; + } + public IASTCompletionContext getCompletionContext() { + return null; + } + public boolean isDeclaration() { + return false; + } + public boolean isDefinition() { + return false; + } + public boolean isReference() { + return false; + } + public char[] toCharArray() { + return fName; + } + public String toString() { + return new String(fName); + } + public void setBinding(IBinding binding) {assert false;} +} + +class ASTBuiltinName extends ASTPreprocessorName { + final private String fFilename; + + public ASTBuiltinName(IASTNode parent, ASTNodeProperty property, String filename, int startNumber, int endNumber, char[] name, IBinding binding) { + super(parent, property, startNumber, endNumber, name, binding); + fFilename= filename; + } + + public boolean contains(IASTNode node) { + return node==this; + } + + public String getContainingFilename() { + return fFilename; + } + + public IASTFileLocation getFileLocation() { + // mstodo Auto-generated method stub + return null; + } + + public IASTNodeLocation[] getNodeLocations() { + // mstodo Auto-generated method stub + return null; + } + + public int getOffset() { + throw new UnsupportedOperationException(); + } + + public String getRawSignature() { + return toString(); + } +} + +class ASTMacroReferenceName extends ASTPreprocessorName { + public ASTMacroReferenceName(IASTNode parent, IPreprocessorMacro macro, ImageLocationInfo imgLocationInfo) { + super(parent, IASTTranslationUnit.EXPANSION_NAME, 0, 0, macro.getNameCharArray(), macro); + } + + public String getContainingFilename() { + return getTranslationUnit().getContainingFilename(); + } + + public String getRawSignature() { + return toString(); + } + + public boolean isReference() { + return true; + } + + // mstodo once names support image-locations, return correct ones here. +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ASTPreprocessorNode.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ASTPreprocessorNode.java new file mode 100644 index 00000000000..4a9e6503f98 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ASTPreprocessorNode.java @@ -0,0 +1,374 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import java.util.ArrayList; + +import org.eclipse.cdt.core.dom.ast.ASTNodeProperty; +import org.eclipse.cdt.core.dom.ast.IASTComment; +import org.eclipse.cdt.core.dom.ast.IASTFileLocation; +import org.eclipse.cdt.core.dom.ast.IASTFunctionStyleMacroParameter; +import org.eclipse.cdt.core.dom.ast.IASTName; +import org.eclipse.cdt.core.dom.ast.IASTNode; +import org.eclipse.cdt.core.dom.ast.IASTNodeLocation; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorElifStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorElseStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorEndifStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorErrorStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorFunctionStyleMacroDefinition; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIfStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIfdefStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIfndefStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIncludeStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorMacroDefinition; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorPragmaStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorUndefStatement; +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit; +import org.eclipse.cdt.core.dom.ast.IMacroBinding; +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit.IDependencyTree; +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit.IDependencyTree.IASTInclusionNode; +import org.eclipse.cdt.internal.core.dom.parser.ASTNode; + +/** + * Models various AST-constructs obtained from the preprocessor. + * @since 5.0 + */ +abstract class ASTPreprocessorNode extends ASTNode { + public ASTPreprocessorNode(IASTNode parent, ASTNodeProperty property, int startNumber, int endNumber) { + setParent(parent); + setPropertyInParent(property); + setOffset(startNumber); + setLength(endNumber-startNumber); + } + + protected String getSource(int offset, int length) { + final IASTTranslationUnit tu= getTranslationUnit(); + IASTNodeLocation[] loc=tu.getLocationInfo(offset, length); + return tu.getUnpreprocessedSignature(loc); + } +} + + + +class ASTComment extends ASTPreprocessorNode implements IASTComment { + private final boolean fIsBlockComment; + public ASTComment(IASTTranslationUnit parent, int startNumber, int endNumber, boolean isBlockComment) { + super(parent, IASTTranslationUnit.PREPROCESSOR_STATEMENT, startNumber, endNumber); + fIsBlockComment= isBlockComment; + } + + public char[] getComment() { + return getSource(getOffset(), getLength()).toCharArray(); + } + + public boolean isBlockComment() { + return fIsBlockComment; + } + + public void setComment(char[] comment) { + assert false; + } +} + + +abstract class ASTDirectiveWithCondition extends ASTPreprocessorNode { + private final int fConditionOffset; + private final int fConditionLength; + private final boolean fActive; + public ASTDirectiveWithCondition(IASTTranslationUnit parent, int startNumber, int condNumber, int condEndNumber, int endNumber, boolean active) { + super(parent, IASTTranslationUnit.PREPROCESSOR_STATEMENT, startNumber, endNumber); + fConditionOffset= condNumber; + fConditionLength= condEndNumber-condNumber; + fActive= active; + } + + public boolean taken() { + return fActive; + } + + public String getConditionString() { + return getSource(fConditionOffset, fConditionLength); + } + + public char[] getCondition() { + return getConditionString().toCharArray(); + } +} + +class ASTEndif extends ASTPreprocessorNode implements IASTPreprocessorEndifStatement { + public ASTEndif(IASTTranslationUnit parent, int startNumber, int endNumber) { + super(parent, IASTTranslationUnit.PREPROCESSOR_STATEMENT, startNumber, endNumber); + } +} + +class ASTElif extends ASTDirectiveWithCondition implements IASTPreprocessorElifStatement { + public ASTElif(IASTTranslationUnit parent, int startNumber, int condNumber, int condEndNumber, int endNumber, boolean active) { + super(parent, startNumber, condNumber, condEndNumber, endNumber, active); + } +} + +class ASTElse extends ASTPreprocessorNode implements IASTPreprocessorElseStatement { + private final boolean fActive; + public ASTElse(IASTTranslationUnit parent, int startNumber, int endNumber, boolean active) { + super(parent, IASTTranslationUnit.PREPROCESSOR_STATEMENT, startNumber, endNumber); + fActive= active; + } + public boolean taken() { + return fActive; + } +} + +class ASTIfndef extends ASTDirectiveWithCondition implements IASTPreprocessorIfndefStatement { + public ASTIfndef(IASTTranslationUnit parent, int startNumber, int condNumber, int condEndNumber, int endNumber, boolean active) { + super(parent, startNumber, condNumber, condEndNumber, endNumber, active); + } +} + +class ASTIfdef extends ASTDirectiveWithCondition implements IASTPreprocessorIfdefStatement { + public ASTIfdef(IASTTranslationUnit parent, int startNumber, int condNumber, int condEndNumber, int endNumber, boolean active) { + super(parent, startNumber, condNumber, condEndNumber, endNumber, active); + } +} + +class ASTIf extends ASTDirectiveWithCondition implements IASTPreprocessorIfStatement { + public ASTIf(IASTTranslationUnit parent, int startNumber, int condNumber, int condEndNumber, int endNumber, boolean active) { + super(parent, startNumber, condNumber, condEndNumber, endNumber, active); + } +} + +class ASTError extends ASTDirectiveWithCondition implements IASTPreprocessorErrorStatement { + public ASTError(IASTTranslationUnit parent, int startNumber, int condNumber, int condEndNumber, int endNumber) { + super(parent, startNumber, condNumber, condEndNumber, endNumber, true); + } + + public char[] getMessage() { + return getCondition(); + } +} + +class ASTPragma extends ASTDirectiveWithCondition implements IASTPreprocessorPragmaStatement { + public ASTPragma(IASTTranslationUnit parent, int startNumber, int condNumber, int condEndNumber, int endNumber) { + super(parent, startNumber, condNumber, condEndNumber, endNumber, true); + } + + public char[] getMessage() { + return getCondition(); + } +} + +class ASTInclusionStatement extends ASTPreprocessorNode implements IASTPreprocessorIncludeStatement { + private final IASTName fName; + private final String fPath; + private final boolean fIsActive; + private final boolean fIsResolved; + private final boolean fIsSystemInclude; + + public ASTInclusionStatement(IASTTranslationUnit parent, int startNumber, int nameStartNumber, int nameEndNumber, int endNumber, + char[] headerName, String filePath, boolean userInclude, boolean active) { + super(parent, IASTTranslationUnit.PREPROCESSOR_STATEMENT, startNumber, endNumber); + fName= new ASTPreprocessorName(this, IASTPreprocessorIncludeStatement.INCLUDE_NAME, nameStartNumber, nameEndNumber, headerName, null); + fPath= filePath == null ? "" : filePath; //$NON-NLS-1$ + fIsActive= active; + fIsResolved= filePath != null; + fIsSystemInclude= !userInclude; + } + + public IASTName getName() { + return fName; + } + + public String getPath() { + return fPath; + } + + public boolean isActive() { + return fIsActive; + } + + public boolean isResolved() { + return fIsResolved; + } + + public boolean isSystemInclude() { + return fIsSystemInclude; + } +} + +class ASTMacro extends ASTPreprocessorNode implements IASTPreprocessorMacroDefinition { + private final IASTName fName; + + /** + * Regular constructor. + */ + public ASTMacro(IASTTranslationUnit parent, IMacroBinding macro, + int startNumber, int nameNumber, int nameEndNumber, int expansionNumber, int endNumber) { + super(parent, IASTTranslationUnit.PREPROCESSOR_STATEMENT, startNumber, endNumber); + fName= new ASTPreprocessorName(this, IASTPreprocessorMacroDefinition.MACRO_NAME, nameNumber, nameEndNumber, macro.getNameCharArray(), macro); + } + + /** + * Constructor for built-in macros + */ + public ASTMacro(IASTTranslationUnit parent, IMacroBinding macro, String filename, int nameNumber, int nameEndNumber) { + super(parent, IASTTranslationUnit.PREPROCESSOR_STATEMENT, 0, 0); + fName= new ASTBuiltinName(this, IASTPreprocessorMacroDefinition.MACRO_NAME, filename, nameNumber, nameEndNumber, macro.getNameCharArray(), macro); + } + + protected IMacroBinding getMacro() { + return (IMacroBinding) fName.getBinding(); + } + + public String getExpansion() { + return new String(getMacro().getExpansion()); + } + + public IASTName getName() { + return fName; + } + + public int getRoleForName(IASTName n) { + return (fName == n) ? r_definition : r_unclear; + } + + public void setExpansion(String exp) {assert false;} + public void setName(IASTName name) {assert false;} +} + +class ASTMacroParameter extends ASTNode implements IASTFunctionStyleMacroParameter { + private final String fParameter; + + public ASTMacroParameter(char[] param) { + fParameter= new String(param); + } + + public String getParameter() { + return fParameter; + } + + public void setParameter(String value) {assert false;} +} + +class ASTFunctionMacro extends ASTMacro implements IASTPreprocessorFunctionStyleMacroDefinition { + /** + * Regular constructor. + */ + public ASTFunctionMacro(IASTTranslationUnit parent, IMacroBinding macro, + int startNumber, int nameNumber, int nameEndNumber, int expansionNumber, int endNumber) { + super(parent, macro, startNumber, nameNumber, nameEndNumber, expansionNumber, endNumber); + } + + /** + * Constructor for builtins + */ + public ASTFunctionMacro(IASTTranslationUnit parent, IMacroBinding macro, + String filename, int nameNumber, int nameEndNumber) { + super(parent, macro, filename, nameNumber, nameEndNumber); + } + + public IASTFunctionStyleMacroParameter[] getParameters() { + FunctionStyleMacro macro= (FunctionStyleMacro) getMacro(); + char[][] paramList= macro.getParamList(); + IASTFunctionStyleMacroParameter[] result= new IASTFunctionStyleMacroParameter[paramList.length]; + for (int i = 0; i < result.length; i++) { + result[i]= new ASTMacroParameter(paramList[i]); + } + return result; + } + + public void addParameter(IASTFunctionStyleMacroParameter parm) {assert false;} +} + + +class ASTUndef extends ASTPreprocessorNode implements IASTPreprocessorUndefStatement { + private final IASTName fName; + public ASTUndef(IASTTranslationUnit parent, char[] name, int startNumber, int nameNumber, int nameEndNumber, int endNumber) { + super(parent, IASTTranslationUnit.PREPROCESSOR_STATEMENT, startNumber, endNumber); + fName= new ASTPreprocessorName(this, IASTPreprocessorUndefStatement.MACRO_NAME, nameNumber, nameEndNumber, name, null); + } + + public IASTName getMacroName() { + return fName; + } +} + +class ASTInclusionNode implements IASTInclusionNode { + protected LocationCtx fLocationCtx; + private IASTInclusionNode[] fInclusions; + + public ASTInclusionNode(LocationCtx ctx) { + fLocationCtx= ctx; + } + + public IASTPreprocessorIncludeStatement getIncludeDirective() { + return fLocationCtx.getInclusionStatement(); + } + + public IASTInclusionNode[] getNestedInclusions() { + if (fInclusions == null) { + ArrayList result= new ArrayList(); + fLocationCtx.getInclusions(result); + fInclusions= (IASTInclusionNode[]) result.toArray(new IASTInclusionNode[result.size()]); + } + return fInclusions; + } +} + +class DependencyTree extends ASTInclusionNode implements IDependencyTree { + public DependencyTree(LocationCtx ctx) { + super(ctx); + } + + public IASTInclusionNode[] getInclusions() { + return getNestedInclusions(); + } + + public String getTranslationUnitPath() { + return fLocationCtx.getFilename(); + } +} + +class ASTFileLocation implements IASTFileLocation { + private String fFilename; + private int fOffset; + private int fLength; + + public ASTFileLocation(String filename, int startOffset, int length) { + fFilename= filename; + fOffset= startOffset; + fLength= length; + } + + public String getFileName() { + return fFilename; + } + + public IASTFileLocation asFileLocation() { + return this; + } + + public int getNodeLength() { + return fLength; + } + + public int getNodeOffset() { + return fOffset; + } + + public int getEndingLineNumber() { + return 0; + } + + public int getStartingLineNumber() { + return 0; + } +} + + diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ASTProblem.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ASTProblem.java new file mode 100644 index 00000000000..1f759cf1c3b --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ASTProblem.java @@ -0,0 +1,235 @@ +/******************************************************************************* + * Copyright (c) 2004, 2007 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM - Initial API and implementation + * Anton Leherbauer (Wind River Systems) + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.Map; + +import org.eclipse.cdt.core.dom.ast.IASTFileLocation; +import org.eclipse.cdt.core.dom.ast.IASTProblem; +import org.eclipse.cdt.internal.core.dom.parser.ASTNode; +import org.eclipse.cdt.internal.core.parser.ParserMessages; + + +/** + * Models the problems found by the preprocessor or lexer. + */ +class ASTProblem extends ASTNode implements IASTProblem { + + private final int id; + private final char[] arg; + + private String message = null; + + public ASTProblem(int id, char[] arg, int startNumber, int endNumber) { + this.id = id; + this.arg = arg; + setOffsetAndLength(startNumber, endNumber-startNumber); + } + + public int getID() { + return id; + } + + public boolean isError() { + return false; + } + + public boolean isWarning() { + return true; + } + + public String getMessage() { + if (message != null) + return message; + + String msg = (String) errorMessages.get(new Integer(id)); + if (msg == null) + msg = ""; //$NON-NLS-1$ + + if (arg != null) { + msg = MessageFormat.format(msg, new Object[] { new String(arg) }); + } + + IASTFileLocation f = getFileLocation(); + String file = null; + int line = 0; + if( f == null ) + { + file = ""; //$NON-NLS-1$ + } else { + file = f.getFileName(); + line = f.getStartingLineNumber(); + } + Object[] args = new Object[] { msg, file, new Integer(line) }; + message = ParserMessages.getFormattedString(PROBLEM_PATTERN, args); + return message; + } + + public boolean checkCategory(int bitmask) { + return ((id & bitmask) != 0); + } + + public String getArguments() { + return arg != null ? String.valueOf(arg) : ""; //$NON-NLS-1$ + } + + + protected static final Map errorMessages; + static { + errorMessages = new HashMap(); + errorMessages + .put( + new Integer(IASTProblem.PREPROCESSOR_POUND_ERROR), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.error")); //$NON-NLS-1$ + errorMessages.put(new Integer(IASTProblem.PREPROCESSOR_POUND_WARNING), ParserMessages + .getString("ScannerProblemFactory.error.preproc.warning")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.PREPROCESSOR_INCLUSION_NOT_FOUND), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.inclusionNotFound")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.PREPROCESSOR_DEFINITION_NOT_FOUND), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.definitionNotFound")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.PREPROCESSOR_INVALID_MACRO_DEFN), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.invalidMacroDefn")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.PREPROCESSOR_INVALID_MACRO_REDEFN), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.invalidMacroRedefn")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.PREPROCESSOR_UNBALANCE_CONDITION), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.unbalancedConditional")); //$NON-NLS-1$ + errorMessages + .put( + new Integer( + IASTProblem.PREPROCESSOR_CONDITIONAL_EVAL_ERROR), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.conditionalEval")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.PREPROCESSOR_MACRO_USAGE_ERROR), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.macroUsage")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.PREPROCESSOR_CIRCULAR_INCLUSION), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.circularInclusion")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.PREPROCESSOR_INVALID_DIRECTIVE), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.invalidDirective")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.PREPROCESSOR_MACRO_PASTING_ERROR), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.macroPasting")); //$NON-NLS-1$ + errorMessages + .put( + new Integer( + IASTProblem.PREPROCESSOR_MISSING_RPAREN_PARMLIST), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.missingRParen")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.PREPROCESSOR_INVALID_VA_ARGS), + ParserMessages + .getString("ScannerProblemFactory.error.preproc.invalidVaArgs")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_INVALID_ESCAPECHAR), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.invalidEscapeChar")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_UNBOUNDED_STRING), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.unboundedString")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_BAD_FLOATING_POINT), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.badFloatingPoint")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_BAD_HEX_FORMAT), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.badHexFormat")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_BAD_OCTAL_FORMAT), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.badOctalFormat")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_BAD_DECIMAL_FORMAT), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.badDecimalFormat")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_ASSIGNMENT_NOT_ALLOWED), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.assignmentNotAllowed")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_DIVIDE_BY_ZERO), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.divideByZero")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_MISSING_R_PAREN), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.missingRParen")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_EXPRESSION_SYNTAX_ERROR), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.expressionSyntaxError")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_ILLEGAL_IDENTIFIER), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.illegalIdentifier")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_BAD_CONDITIONAL_EXPRESSION), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.badConditionalExpression")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_UNEXPECTED_EOF), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.unexpectedEOF")); //$NON-NLS-1$ + errorMessages + .put( + new Integer(IASTProblem.SCANNER_BAD_CHARACTER), + ParserMessages + .getString("ScannerProblemFactory.error.scanner.badCharacter")); //$NON-NLS-1$ + errorMessages.put(new Integer(IASTProblem.SYNTAX_ERROR), ParserMessages + .getString("ParserProblemFactory.error.syntax.syntaxError")); //$NON-NLS-1$ + } + + protected final static String PROBLEM_PATTERN = "BaseProblemFactory.problemPattern"; //$NON-NLS-1$ +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java new file mode 100644 index 00000000000..c6e9793e46b --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java @@ -0,0 +1,2015 @@ +/******************************************************************************* + * Copyright (c) 2004, 2007 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM - Initial API and implementation + * Anton Leherbauer (Wind River Systems) + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import java.io.File; +import java.io.IOException; +import java.util.Calendar; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.eclipse.cdt.core.dom.ICodeReaderFactory; +import org.eclipse.cdt.core.dom.parser.IScannerExtensionConfiguration; +import org.eclipse.cdt.core.parser.CodeReader; +import org.eclipse.cdt.core.parser.EndOfFileException; +import org.eclipse.cdt.core.parser.IExtendedScannerInfo; +import org.eclipse.cdt.core.parser.IMacro; +import org.eclipse.cdt.core.parser.IParserLogService; +import org.eclipse.cdt.core.parser.IPreprocessorDirective; +import org.eclipse.cdt.core.parser.IProblem; +import org.eclipse.cdt.core.parser.IScanner; +import org.eclipse.cdt.core.parser.IScannerInfo; +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.Keywords; +import org.eclipse.cdt.core.parser.OffsetLimitReachedException; +import org.eclipse.cdt.core.parser.ParseError; +import org.eclipse.cdt.core.parser.ParserLanguage; +import org.eclipse.cdt.core.parser.ast.IASTFactory; +import org.eclipse.cdt.core.parser.util.CharArrayIntMap; +import org.eclipse.cdt.core.parser.util.CharArrayObjectMap; +import org.eclipse.cdt.core.parser.util.CharArraySet; +import org.eclipse.cdt.internal.core.parser.scanner.ExpressionEvaluator.EvalException; +import org.eclipse.cdt.internal.core.parser.scanner.Lexer.LexerOptions; +import org.eclipse.cdt.internal.core.parser.scanner.MacroDefinitionParser.InvalidMacroDefinitionException; +import org.eclipse.cdt.internal.core.parser.scanner2.IIndexBasedCodeReaderFactory; +import org.eclipse.cdt.internal.core.parser.scanner2.ScannerUtility; + +/** + * C-Preprocessor providing tokens for the parsers. The class should not be used directly, rather than that + * you should be using the {@link IScanner} interface. + * @since 5.0 + */ +public class CPreprocessor implements ILexerLog, IScanner { + public static final int tDEFINED= IToken.FIRST_RESERVED_PREPROCESSOR; + + private static final int ORIGIN_PREPROCESSOR_DIRECTIVE = OffsetLimitReachedException.ORIGIN_PREPROCESSOR_DIRECTIVE; + private static final int ORIGIN_INACTIVE_CODE = OffsetLimitReachedException.ORIGIN_INACTIVE_CODE; +// private static final int ORIGIN_MACRO_EXPANSION = OffsetLimitReachedException.ORIGIN_MACRO_EXPANSION; + + private static final char[] EMPTY_CHAR_ARRAY = new char[0]; + private static final char[] ONE = "1".toCharArray(); //$NON-NLS-1$ + private static final String EMPTY_STRING = ""; //$NON-NLS-1$ + private static final EndOfFileException EOF = new EndOfFileException(); + + + // standard built-ins + private static final ObjectStyleMacro __cplusplus = new ObjectStyleMacro("__cplusplus".toCharArray(), ONE); //$NON-NLS-1$ + private static final ObjectStyleMacro __STDC__ = new ObjectStyleMacro("__STDC__".toCharArray(), ONE); //$NON-NLS-1$ + private static final ObjectStyleMacro __STDC_HOSTED__ = new ObjectStyleMacro("__STDC_HOSTED_".toCharArray(), ONE); //$NON-NLS-1$ + private static final ObjectStyleMacro __STDC_VERSION__ = new ObjectStyleMacro("__STDC_VERSION_".toCharArray(), "199901L".toCharArray()); //$NON-NLS-1$ //$NON-NLS-2$ + + private static final ILexerLog LEXERLOG_NULL= new ILexerLog() { + public void handleComment(boolean isBlockComment, int offset, int endOffset) {} + public void handleProblem(int problemID, char[] source, int offset, int endOffset) {} + }; + + private interface IIncludeFileTester { + Object checkFile(String path, String fileName); + } + + final private IIncludeFileTester createCodeReaderTester= new IIncludeFileTester() { + public Object checkFile(String path, String fileName) { + return createReader(path, fileName); + } + }; + + final private IIncludeFileTester createPathTester= new IIncludeFileTester() { + public Object checkFile(String path, String fileName) { + path= ScannerUtility.createReconciledPath(path, fileName); + if (new File(path).exists()) { + return path; + } + return null; + } + }; + + // standard built-ins + final private DynamicStyleMacro __FILE__= new DynamicStyleMacro("__FILE__".toCharArray()) { //$NON-NLS-1$ + public Token execute() { + StringBuffer buffer = new StringBuffer("\""); //$NON-NLS-1$ + buffer.append(getCurrentFilename()); + buffer.append('\"'); + return new ImageToken(IToken.tSTRING, 0, 0, buffer.toString().toCharArray()); + } + }; + final private DynamicStyleMacro __DATE__= new DynamicStyleMacro("__DATE__".toCharArray()) { //$NON-NLS-1$ + final private void append(StringBuffer buffer, int value) { + if (value < 10) + buffer.append("0"); //$NON-NLS-1$ + buffer.append(value); + } + + public Token execute() { + StringBuffer buffer = new StringBuffer("\""); //$NON-NLS-1$ + Calendar cal = Calendar.getInstance(); + buffer.append(cal.get(Calendar.MONTH)); + buffer.append(" "); //$NON-NLS-1$ + append(buffer, cal.get(Calendar.DAY_OF_MONTH)); + buffer.append(" "); //$NON-NLS-1$ + buffer.append(cal.get(Calendar.YEAR)); + buffer.append("\""); //$NON-NLS-1$ + return new ImageToken(IToken.tSTRING, 0, 0, buffer.toString().toCharArray()); + } + }; + + final private DynamicStyleMacro __TIME__ = new DynamicStyleMacro("__TIME__".toCharArray()) { //$NON-NLS-1$ + final private void append(StringBuffer buffer, int value) { + if (value < 10) + buffer.append("0"); //$NON-NLS-1$ + buffer.append(value); + } + + public Token execute() { + StringBuffer buffer = new StringBuffer("\""); //$NON-NLS-1$ + Calendar cal = Calendar.getInstance(); + append(buffer, cal.get(Calendar.HOUR)); + buffer.append(":"); //$NON-NLS-1$ + append(buffer, cal.get(Calendar.MINUTE)); + buffer.append(":"); //$NON-NLS-1$ + append(buffer, cal.get(Calendar.SECOND)); + buffer.append("\""); //$NON-NLS-1$ + return new ImageToken(IToken.tSTRING, 0, 0, buffer.toString().toCharArray()); + } + }; + + final private DynamicStyleMacro __LINE__ = new DynamicStyleMacro("__LINE__".toCharArray()) { //$NON-NLS-1$ + public Token execute() { + int lineNumber= fLocationMap.getCurrentLineNumber(fCurrentContext.currentPPToken().getOffset()); + return new ImageToken(IToken.tINTEGER, 0, 0, Long.toString(lineNumber).toCharArray()); + } + }; + + final private IParserLogService fLog; + final private ICodeReaderFactory fCodeReaderFactory; + private final ExpressionEvaluator fExpressionEvaluator; + private final MacroDefinitionParser fMacroDefinitionParser= new MacroDefinitionParser(); + + // configuration + final private ParserLanguage fLanguage; + final private LexerOptions fLexOptions= new LexerOptions(); + final private boolean fCheckNumbers; + final private char[] fAdditionalNumericLiteralSuffixes; + final private CharArrayIntMap fKeywords; + final private CharArrayIntMap fPPKeywords; + final private String[] fIncludePaths; + final private String[] fQuoteIncludePaths; + + private int fContentAssistLimit; + + // state information + private final CharArrayObjectMap fMacroDictionary = new CharArrayObjectMap(512); + private final LocationMap fLocationMap = new LocationMap(); + + /** Set of already included files */ + private final CharArraySet includedFiles= new CharArraySet(32); + private int fTokenCount; + + private final Lexer fRootLexer; + private final ScannerContext fRootContext; + private ScannerContext fCurrentContext; + + private boolean isCancelled = false; + + private Token fPrefetchedToken; + private Token fLastToken; + + + + public CPreprocessor(CodeReader reader, IScannerInfo info, ParserLanguage language, IParserLogService log, + IScannerExtensionConfiguration configuration, ICodeReaderFactory readerFactory) { + fLanguage= language; + fLog = log; + fCheckNumbers= true; // mstodo room for optimization. + fAdditionalNumericLiteralSuffixes= nonNull(configuration.supportAdditionalNumericLiteralSuffixes()); + fLexOptions.fSupportDollarInitializers= configuration.support$InIdentifiers(); + fLexOptions.fSupportMinAndMax = configuration.supportMinAndMaxOperators(); + fKeywords= new CharArrayIntMap(40, -1); + fPPKeywords= new CharArrayIntMap(40, -1); + configureKeywords(language, configuration); + + fIncludePaths= info.getIncludePaths(); + fQuoteIncludePaths= getQuoteIncludePath(info); + + fExpressionEvaluator= new ExpressionEvaluator(); + fCodeReaderFactory= readerFactory; + + setupMacroDictionary(configuration, info); + + ILocationCtx ctx= fLocationMap.pushTranslationUnit(new String(reader.filename), reader.buffer); + fRootLexer= new Lexer(reader.buffer, (LexerOptions) fLexOptions.clone(), this); + fRootContext= fCurrentContext= new ScannerContextFile(ctx, null, fRootLexer); + + if (info instanceof IExtendedScannerInfo) { + final IExtendedScannerInfo einfo= (IExtendedScannerInfo) info; + + // files provided on command line (-imacros, -include) + registerPreIncludedFiles(einfo.getMacroFiles(), einfo.getIncludeFiles()); + } + } + + public void setContentAssistMode(int offset) { + fRootLexer.setContentAssistMode(offset); + } + + + // mstodo keywords should be provided directly by the language + private void configureKeywords(ParserLanguage language, IScannerExtensionConfiguration configuration) { + if (language == ParserLanguage.C) { + Keywords.addKeywordsC(fKeywords); + } + else { + Keywords.addKeywordsCpp(fKeywords); + } + CharArrayIntMap additionalKeywords= configuration.getAdditionalKeywords(); + if (additionalKeywords != null) { + fKeywords.putAll(additionalKeywords); + } + additionalKeywords= configuration.getAdditionalPreprocessorKeywords(); + if (additionalKeywords != null) { + fPPKeywords.putAll(additionalKeywords); + } + } + + protected String getCurrentFilename() { + return fLocationMap.getCurrentFilename(); + } + + private char[] nonNull(char[] array) { + return array == null ? EMPTY_CHAR_ARRAY : array; + } + + private String[] getQuoteIncludePath(IScannerInfo info) { + if (info instanceof IExtendedScannerInfo) { + final IExtendedScannerInfo einfo= (IExtendedScannerInfo) info; + final String[] qip= einfo.getLocalIncludePath(); + if (qip != null && qip.length > 0) { + final String[] result= new String[qip.length + fIncludePaths.length]; + System.arraycopy(qip, 0, result, 0, qip.length); + System.arraycopy(fIncludePaths, 0, result, qip.length, fIncludePaths.length); + return result; + } + } + return info.getIncludePaths(); + } + + private void setupMacroDictionary(IScannerExtensionConfiguration config, IScannerInfo info) { + // built in macros + fMacroDictionary.put(__STDC__.getNameCharArray(), __STDC__); + fMacroDictionary.put(__FILE__.getNameCharArray(), __FILE__); + fMacroDictionary.put(__DATE__.getNameCharArray(), __DATE__); + fMacroDictionary.put(__TIME__.getNameCharArray(), __TIME__); + fMacroDictionary.put(__LINE__.getNameCharArray(), __LINE__); + + if (fLanguage == ParserLanguage.CPP) + fMacroDictionary.put(__cplusplus.getNameCharArray(), __cplusplus); + else { + fMacroDictionary.put(__STDC_HOSTED__.getNameCharArray(), __STDC_HOSTED__); + fMacroDictionary.put(__STDC_VERSION__.getNameCharArray(), __STDC_VERSION__); + } + + CharArrayObjectMap toAdd = config.getAdditionalMacros(); + for (int i = 0; i < toAdd.size(); ++i) { + addDefinition((IMacro) toAdd.getAt(i)); + } + + // macros provided on command-line (-D) + final boolean initEmptyMacros= config.initializeMacroValuesTo1(); + final Map macroDict= info.getDefinedSymbols(); + if (macroDict != null) { + for (Iterator iterator = macroDict.entrySet().iterator(); iterator.hasNext();) { + final Map.Entry entry = (Map.Entry) iterator.next(); + final String key= (String) entry.getKey(); + final String value= ((String) entry.getValue()).trim(); + if (initEmptyMacros && value.length() == 0) { + addMacroDefinition(key.toCharArray(), ONE); + } + else { + addMacroDefinition(key.toCharArray(), value.toCharArray()); + } + } + } + + Object[] predefined= fMacroDictionary.valueArray(); + for (int i = 0; i < predefined.length; i++) { + fLocationMap.registerPredefinedMacro((PreprocessorMacro) predefined[i]); + } + } + + private void registerPreIncludedFiles(final String[] macroFiles, final String[] preIncludedFiles) { + if (preIncludedFiles != null && preIncludedFiles.length > 0) { + final char[] buffer= createSyntheticFile(preIncludedFiles); + ILocationCtx ctx= fLocationMap.pushPreInclusion(buffer, 0, false); + fCurrentContext= new ScannerContextFile(ctx, fCurrentContext, new Lexer(buffer, fLexOptions, this)); + } + + if (macroFiles != null && macroFiles.length > 0) { + final char[] buffer= createSyntheticFile(macroFiles); + ILocationCtx ctx= fLocationMap.pushPreInclusion(buffer, 0, true); + fCurrentContext= new ScannerContextMacroFile(this, ctx, fCurrentContext, new Lexer(buffer, fLexOptions, this)); + } + } + + private char[] createSyntheticFile(String[] files) { + int totalLength= 0; + final char[] instruction= "#include <".toCharArray(); //$NON-NLS-1$ + for (int i = 0; i < files.length; i++) { + totalLength+= instruction.length + 2 + files[i].length(); + } + final char[] buffer= new char[totalLength]; + int pos= 0; + for (int i = 0; i < files.length; i++) { + final char[] fileName= files[i].toCharArray(); + System.arraycopy(instruction, 0, buffer, pos, instruction.length); + pos+= instruction.length; + System.arraycopy(fileName, 0, buffer, pos, fileName.length); + pos+= fileName.length; + buffer[pos++]= '>'; + buffer[pos++]= '\n'; + } + return buffer; + } + +// private boolean isCircularInclusion(InclusionData data) { +// // mstodo +// for (int i = 0; i < bufferStackPos; ++i) { +// if (bufferData[i] instanceof CodeReader +// && CharArrayUtils.equals( +// ((CodeReader) bufferData[i]).filename, +// data.reader.filename)) { +// return true; +// } else if (bufferData[i] instanceof InclusionData +// && CharArrayUtils +// .equals( +// ((InclusionData) bufferData[i]).reader.filename, +// data.reader.filename)) { +// return true; +// } +// } +// return false; +// } + + + + /** + * Check if the given inclusion was already included before. + * + * @param inclusionData + * @return + */ +// private boolean isRepeatedInclusion(InclusionData inclusionData) { +// return includedFiles.containsKey(inclusionData.reader.filename); +// } + + public PreprocessorMacro addMacroDefinition(char[] key, char[] value) { + final Lexer lex= new Lexer(key, fLexOptions, LEXERLOG_NULL); + try { + PreprocessorMacro result= fMacroDefinitionParser.parseMacroDefinition(lex, LEXERLOG_NULL, value); + fLocationMap.registerPredefinedMacro(result); + fMacroDictionary.put(result.getNameCharArray(), result); + return result; + } + catch (Exception e) { + fLog.traceLog("Invalid macro definition: '" + String.valueOf(key) + "'"); //$NON-NLS-1$//$NON-NLS-2$ + return null; + } + } + + public int getCount() { + return fTokenCount; + } + + public Map getDefinitions() { + CharArrayObjectMap objMap = getRealDefinitions(); + int size = objMap.size(); + Map hashMap = new HashMap(size); + for (int i = 0; i < size; i++) { + hashMap.put(String.valueOf(objMap.keyAt(i)), objMap.getAt(i)); + } + + return hashMap; + } + + public CharArrayObjectMap getRealDefinitions() { + return fMacroDictionary; + } + + public String[] getIncludePaths() { + return fIncludePaths; + } + + public boolean isOnTopContext() { + return fCurrentContext == fRootContext; + } + + public synchronized void cancel() { + isCancelled= true; + } + + /** + * Returns next token for the parser. + * @throws OffsetLimitReachedException + */ + public IToken nextToken() throws EndOfFileException { + if (isCancelled) { + throw new ParseError(ParseError.ParseErrorKind.TIMEOUT_OR_CANCELLED); + } + + // use prefetched token or get a new one. + Token t1= fPrefetchedToken; + if (t1 == null) { + t1= fetchTokenFromPreprocessor(); + } + else { + fPrefetchedToken= null; + } + + final int tt1= t1.getType(); + switch(tt1) { + case Lexer.tEND_OF_INPUT: + if (fContentAssistLimit < 0) { + throw EOF; + } + t1= new SimpleToken(IToken.tEOC, fContentAssistLimit, fContentAssistLimit); + break; + case IToken.tSTRING: + case IToken.tLSTRING: + boolean isWide= tt1 == IToken.tLSTRING; + Token t2; + StringBuffer buf= null; + int endOffset= 0; + loop: while(true) { + t2= fetchTokenFromPreprocessor(); + final int tt2= t2.getType(); + switch(tt2) { + case IToken.tLSTRING: + isWide= true; + // no break; + case IToken.tSTRING: + if (buf == null) { + buf= new StringBuffer(); + appendStringContent(buf, t1); + } + appendStringContent(buf, t2); + endOffset= t2.getEndOffset(); + continue loop; + + default: + break loop; + } + } + fPrefetchedToken= t2; + if (buf != null) { + char[] image= new char[buf.length() + (isWide ? 3 : 2)]; + int off= -1; + if (isWide) { + image[++off]= 'L'; + } + image[++off]= '"'; + buf.getChars(0, buf.length(), image, ++off); + image[image.length-1]= '"'; + t1= new ImageToken((isWide ? IToken.tLSTRING : IToken.tSTRING), t1.getOffset(), endOffset, image); + } + } + + if (fLastToken != null) { + fLastToken.setNext(t1); + } + fLastToken= t1; + return t1; + } + + private void appendStringContent(StringBuffer buf, Token t1) { + final char[] image= t1.getCharImage(); + final int start= image[0]=='"' ? 1 : 2; + buf.append(image, start, image.length-start); + } + + Token fetchTokenFromPreprocessor() throws OffsetLimitReachedException { + ++fTokenCount; + Token ppToken= fCurrentContext.currentPPToken(); + while(true) { + switch(ppToken.getType()) { + case Lexer.tBEFORE_INPUT: + case Lexer.tNEWLINE: + ppToken= fCurrentContext.nextPPToken(); + continue; + + case Lexer.tEND_OF_INPUT: + final ILocationCtx locationCtx = fCurrentContext.getLocationCtx(); + if (locationCtx != null) { + fLocationMap.popContext(locationCtx); + } + fCurrentContext= fCurrentContext.getParent(); + if (fCurrentContext == null) { + return ppToken; + } + + ppToken= fCurrentContext.currentPPToken(); + continue; + + case IToken.tPOUND: + final Lexer lexer= fCurrentContext.getLexerForPPDirective(); + if (lexer != null) { + executeDirective(lexer, ppToken.getOffset()); + continue; + } + break; + + case IToken.tIDENTIFIER: + if (fCurrentContext.expandsMacros() && expandMacro(ppToken, true)) { + continue; + } + + final char[] name= ppToken.getCharImage(); + int tokenType = fKeywords.get(name); + if (tokenType != fKeywords.undefined) { + ppToken.setType(tokenType); + } + break; + + case IToken.tINTEGER: + if (fCheckNumbers) { + checkNumber(ppToken, false); + } + break; + + case IToken.tFLOATINGPT: + if (fCheckNumbers) { + checkNumber(ppToken, true); + } + break; + } + fCurrentContext.nextPPToken(); + return ppToken; + } + } + + private void checkNumber(Token number, final boolean isFloat) { + final char[] image= number.getCharImage(); + boolean hasExponent = false; + + boolean isHex = false; + boolean isOctal = false; + boolean hasDot= false; + + int pos= 0; + if (image.length > 1) { + if (image[0] == '0') { + switch (image[++pos]) { + case 'x': + case 'X': + isHex = true; + ++pos; + break; + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': + isOctal = true; + ++pos; + break; + case '8': case '9': + handleProblem(IProblem.SCANNER_BAD_OCTAL_FORMAT, image, number.getOffset(), number.getEndOffset()); + return; + } + } + } + + loop: for (; pos < image.length; pos++) { + switch (image[pos]) { + // octal digits + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': + continue; + + // decimal digits + case '8': case '9': + if (isOctal) { + handleProblem(IProblem.SCANNER_BAD_OCTAL_FORMAT, image, number.getOffset(), number.getEndOffset()); + return; + } + continue; + + // hex digits + case 'a': case 'A': case 'b': case 'B': case 'c': case 'C': case 'd': case 'D': case 'f': case 'F': + if (isHex && !hasExponent) { + continue; + } + break loop; + + case '.': + if (hasDot) { + handleProblem(IProblem.SCANNER_BAD_FLOATING_POINT, image, number.getOffset(), number.getEndOffset()); + return; + } + hasDot= true; + continue; + + // check for exponent or hex digit + case 'E': case 'e': + if (isHex && !hasExponent) { + continue; + } + if (isFloat && !isHex && !hasExponent && pos+1 >= image.length) { + switch (image[pos+1]) { + case '+': case '-': + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + hasExponent = true; + ++pos; + continue; + } + } + break loop; + + // check for hex float exponent + case 'p': case 'P': + if (isFloat && isHex && !hasExponent && pos+1 >= image.length) { + switch (image[pos+1]) { + case '+': case '-': + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + hasExponent = true; + ++pos; + continue; + } + } + break loop; + + default: + break loop; + } + } + + // check the suffix + loop: for (; pos < image.length; pos++) { + final char c= image[pos]; + switch (c) { + case 'u': case 'U': case 'L': case 'l': + continue; + case 'f': case 'F': + if (isFloat) { + continue loop; + } + } + for (int i=0; i + * Checks whether the identifier causes a macro expansion. May advance the current lexer + * to check for the opening bracket succeeding the identifier. + *

+ * If applicable the macro is expanded and the resulting tokens are put into a scanner context. + * @param identifier the token where macro expansion may occur. + * @param multiline whether we are allowed to check subsequent lines for macro arguments. + * @return + * @throws OffsetLimitReachedException + */ + private boolean expandMacro(final Token identifier, boolean multiline) throws OffsetLimitReachedException { + final char[] name= identifier.getCharImage(); + PreprocessorMacro macro= (PreprocessorMacro) fMacroDictionary.get(name); + if (macro == null) { + return false; + } + + Token bracket= null; + if (macro instanceof FunctionStyleMacro) { + bracket= fCurrentContext.nextPPToken(); + if (multiline) { + while(bracket.getType() == Lexer.tNEWLINE) { + bracket= fCurrentContext.nextPPToken(); + } + } + if (bracket.getType() != IToken.tLPAREN) { + return false; + } + } + + // mstodo really expand the macro + return true; + } + + public Object getAdapter(Class adapter) { + if (adapter.isAssignableFrom(fLocationMap.getClass())) { + return fLocationMap; + } + return null; + } + + // stuff to be removed + public IMacro addDefinition(char[] key, char[] value) { + throw new UnsupportedOperationException(); + } + public void setScanComments(boolean val) { + throw new UnsupportedOperationException(); + } + public char[] getMainFilename() { + throw new UnsupportedOperationException(); + } + public void addDefinition(IMacro macro) { + throw new UnsupportedOperationException(); + } + public IMacro addDefinition(char[] name, char[][] params, char[] expansion) { + throw new UnsupportedOperationException(); + } + public void setASTFactory(IASTFactory f) { + throw new UnsupportedOperationException(); + } + public org.eclipse.cdt.internal.core.parser.scanner2.ILocationResolver getLocationResolver() { + throw new UnsupportedOperationException(); + } + public void setOffsetBoundary(int offset) { + throw new UnsupportedOperationException(); + } + +// private int skipOverMacroArg() { +// char[] buffer = bufferStack[bufferStackPos]; +// int limit = bufferLimit[bufferStackPos]; +// int argEnd = bufferPos[bufferStackPos]--; +// int nesting = 0; +// while (++bufferPos[bufferStackPos] < limit) { +// switch (buffer[bufferPos[bufferStackPos]]) { +// case '(': +// ++nesting; +// break; +// case ')': +// if (nesting == 0) { +// --bufferPos[bufferStackPos]; +// return argEnd; +// } +// --nesting; +// break; +// case ',': +// if (nesting == 0) { +// --bufferPos[bufferStackPos]; +// return argEnd; +// } +// break; +// // fix for 95119 +// case '\'': +// boolean escapedChar = false; +// loop: while (++bufferPos[bufferStackPos] < bufferLimit[bufferStackPos]) { +// switch (buffer[bufferPos[bufferStackPos]]) { +// case '\\': +// escapedChar = !escapedChar; +// continue; +// case '\'': +// if (escapedChar) { +// escapedChar = false; +// continue; +// } +// break loop; +// default: +// escapedChar = false; +// } +// } +// break; +// case '"': +// boolean escaped = false; +// loop: while (++bufferPos[bufferStackPos] < bufferLimit[bufferStackPos]) { +// switch (buffer[bufferPos[bufferStackPos]]) { +// case '\\': +// escaped = !escaped; +// continue; +// case '"': +// if (escaped) { +// escaped = false; +// continue; +// } +// break loop; +// default: +// escaped = false; +// } +// } +// break; +// } +// argEnd = bufferPos[bufferStackPos]; +// skipOverWhiteSpace(); +// } +// --bufferPos[bufferStackPos]; +// // correct argEnd when reaching limit, (bug 179383) +// if (argEnd==limit) { +// argEnd--; +// } +// return argEnd; +// } +// +// +// private char[] expandFunctionStyleMacro(FunctionStyleMacro macro) { +// char[] buffer = bufferStack[bufferStackPos]; +// int limit = bufferLimit[bufferStackPos]; +// int start = bufferPos[bufferStackPos] - macro.name.length + 1; +// skipOverWhiteSpace(); +// while (bufferPos[bufferStackPos] < limit +// && buffer[bufferPos[bufferStackPos]] == '\\' +// && bufferPos[bufferStackPos] + 1 < buffer.length +// && buffer[bufferPos[bufferStackPos] + 1] == '\n') { +// bufferPos[bufferStackPos] += 2; +// skipOverWhiteSpace(); +// } +// +// if (++bufferPos[bufferStackPos] >= limit) { +// //allow a macro boundary cross here, but only if the caller was +// // prepared to accept a bufferStackPos change +// if (pushContext) { +// int idx = -1; +// int stackpPos = bufferStackPos; +// while (bufferData[stackpPos] != null +//// && bufferData[stackpPos] instanceof MacroData +// ) { +// stackpPos--; +// if (stackpPos < 0) +// return EMPTY_CHAR_ARRAY; +// idx = indexOfNextNonWhiteSpace(bufferStack[stackpPos], +// bufferPos[stackpPos], bufferLimit[stackpPos]); +// if (idx >= bufferLimit[stackpPos]) +// continue; +// if (idx > 0 && bufferStack[stackpPos][idx] == '(') +// break; +// bufferPos[bufferStackPos]--; +// return null; +// } +// if (idx == -1) { +// bufferPos[bufferStackPos]--; +// return null; +// } +// +//// MacroData data; +// IMacro popMacro= macro; +// do { +//// data= (MacroData) bufferData[bufferStackPos]; +// popContextForFunctionMacroName(popMacro); +// popMacro= data.macro; +// } while (bufferStackPos > stackpPos); +// +// bufferPos[bufferStackPos] = idx; +// buffer = bufferStack[bufferStackPos]; +// limit = bufferLimit[bufferStackPos]; +// start = data.startOffset; +// } else { +// bufferPos[bufferStackPos]--; +// return null; +// } +// } +// +// // fix for 107150: the scanner stops at the \n or \r after skipOverWhiteSpace() take that into consideration +// while (bufferPos[bufferStackPos] + 1 < limit && (buffer[bufferPos[bufferStackPos]] == '\n' || buffer[bufferPos[bufferStackPos]] == '\r')) { +// bufferPos[bufferStackPos]++; // skip \n or \r +// skipOverWhiteSpace(); // skip any other spaces after the \n +// +// if (bufferPos[bufferStackPos] + 1 < limit && buffer[bufferPos[bufferStackPos]] != '(' && buffer[bufferPos[bufferStackPos] + 1] == '(') +// bufferPos[bufferStackPos]++; // advance to ( if necessary +// } +// +// if (buffer[bufferPos[bufferStackPos]] != '(') { +// bufferPos[bufferStackPos]--; +// return null; +// } +// +// char[][] arglist = macro.arglist; +// int currarg = 0; +// CharArrayObjectMap argmap = new CharArrayObjectMap(arglist.length); +// +// while (bufferPos[bufferStackPos] < limit) { +// skipOverWhiteSpace(); +// +// if (bufferPos[bufferStackPos] + 1 >= limit) +// break; +// +// if (buffer[++bufferPos[bufferStackPos]] == ')') { +// if (currarg > 0 && argmap.size() <= currarg) { +// argmap.put(arglist[currarg], EMPTY_CHAR_ARRAY); +// } +// break; // end of macro +// } +// if (buffer[bufferPos[bufferStackPos]] == ',') { +// if (argmap.size() <= currarg) { +// argmap.put(arglist[currarg], EMPTY_CHAR_ARRAY); +// } +// currarg++; +// continue; +// } +// +// if ((currarg >= arglist.length || arglist[currarg] == null) +// && !macro.hasVarArgs() && !macro.hasGCCVarArgs()) { +// // too many args and no variable argument +// handleProblem(IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, +// bufferPos[bufferStackPos], macro.name); +// break; +// } +// +// int argstart = bufferPos[bufferStackPos]; +// +// int argend = -1; +// if ((macro.hasGCCVarArgs() || macro.hasVarArgs()) && currarg == macro.getVarArgsPosition()) { +// // there are varargs and the other parameters have been accounted +// // for, the rest will replace __VA_ARGS__ or name where +// // "name..." is the parameter +// for (;;) { +// argend= skipOverMacroArg(); +// skipOverWhiteSpace(); +// // to continue we need at least a comma and another char. +// if (bufferPos[bufferStackPos]+2 >= limit) { +// break; +// } +// if (buffer[++bufferPos[bufferStackPos]] == ')') { +// bufferPos[bufferStackPos]--; +// break; +// } +// // it's a comma +// bufferPos[bufferStackPos]++; +// } +// } else { +// argend = skipOverMacroArg(); +// } +// +// char[] arg = EMPTY_CHAR_ARRAY; +// int arglen = argend - argstart + 1; +// if (arglen > 0) { +// arg = new char[arglen]; +// System.arraycopy(buffer, argstart, arg, 0, arglen); +// } +// +// argmap.put(arglist[currarg], arg); +// } +// +// int numRequiredArgs = arglist.length; +// for (int i = 0; i < arglist.length; i++) { +// if (arglist[i] == null) { +// numRequiredArgs = i; +// break; +// } +// } +// +// /* Don't require a match for the vararg placeholder */ +// /* Workaround for bugzilla 94365 */ +// if (macro.hasGCCVarArgs()|| macro.hasVarArgs()) +// numRequiredArgs--; +// +// if (argmap.size() < numRequiredArgs) { +// handleProblem(IProblem.PREPROCESSOR_MACRO_USAGE_ERROR, +// bufferPos[bufferStackPos], macro.name); +// } +// +// char[] result = null; +// if (macro instanceof DynamicFunctionStyleMacro) { +// result = ((DynamicFunctionStyleMacro) macro).execute(argmap); +// } else { +// CharArrayObjectMap replacedArgs = new CharArrayObjectMap(argmap +// .size()); +// int size = expandFunctionStyleMacro(macro.getExpansion(), argmap, +// replacedArgs, null); +// result = new char[size]; +// expandFunctionStyleMacro(macro.getExpansion(), argmap, replacedArgs, +// result); +// } +// if (pushContext) +// { +//// pushContext(result, new FunctionMacroData(start, bufferPos[bufferStackPos] + 1, +//// macro, argmap)); +// } +// return result; +// } +// +// private char[] replaceArgumentMacros(char[] arg) { +// int limit = arg.length; +// int start = -1, end = -1; +// Object expObject = null; +// for (int pos = 0; pos < limit; pos++) { +// char c = arg[pos]; +// if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' +// || Character.isLetter(c) +// || (support$Initializers && c == '$')) { +// start = pos; +// while (++pos < limit) { +// c = arg[pos]; +// if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +// || c == '_' || (c >= '0' && c <= '9') +// || (support$Initializers && c == '$') +// || Character.isUnicodeIdentifierPart(c)) { +// continue; +// } +// break; +// } +// end = pos - 1; +// } +// else if (c == '"') { +// boolean escaped= false; +// while (++pos < limit) { +// c = arg[pos]; +// if (!escaped && c == '"') { +// break; +// } +// if (c == '\\') { +// escaped= !escaped; +// } +// else { +// escaped= false; +// } +// } +// } +// else if (c == '\'') { +// boolean escaped= false; +// while (++pos < limit) { +// c = arg[pos]; +// if (!escaped && c == '\'') { +// break; +// } +// if (c == '\\') { +// escaped= !escaped; +// } +// else { +// escaped= false; +// } +// } +// } +// +// if (start != -1 && end >= start) { +// //Check for macro expansion +// expObject = fMacroDictionary.get(arg, start, (end - start + 1)); +// if (expObject == null || !shouldExpandMacro((IMacro) expObject)) { +// expObject = null; +// start = -1; +// continue; +// } +// //else, break and expand macro +// break; +// } +// } +// +// if (expObject == null) +// { +// return arg; +// } +// +// +// char[] expansion = null; +// if (expObject instanceof FunctionStyleMacro) { +// FunctionStyleMacro expMacro = (FunctionStyleMacro) expObject; +// pushContext((start == 0) ? arg : CharArrayUtils.extract(arg, start, +// arg.length - start)); +// bufferPos[bufferStackPos] += end - start + 1; +// expansion = handleFunctionStyleMacro(expMacro, false); +// end = bufferPos[bufferStackPos] + start; +// popContext(); +// } else if (expObject instanceof ObjectStyleMacro) { +// ObjectStyleMacro expMacro = (ObjectStyleMacro) expObject; +// expansion = expMacro.getExpansion(); +// } else if (expObject instanceof char[]) { +// expansion = (char[]) expObject; +// } else if (expObject instanceof DynamicStyleMacro) { +// DynamicStyleMacro expMacro = (DynamicStyleMacro) expObject; +// expansion = expMacro.execute(); +// } +// +// if (expansion != null) { +// int newlength = start + expansion.length + (limit - end - 1); +// char[] result = new char[newlength]; +// System.arraycopy(arg, 0, result, 0, start); +// System.arraycopy(expansion, 0, result, start, expansion.length); +// if (arg.length > end + 1) +// System.arraycopy(arg, end + 1, result, +// start + expansion.length, limit - end - 1); +// +// +// beforeReplaceAllMacros(); +// //we need to put the macro on the context stack in order to detect +// // recursive macros +//// pushContext(EMPTY_CHAR_ARRAY, +//// new MacroData(start, start +//// + ((IMacro) expObject).getName().length, +//// (IMacro) expObject) +//// ); +// arg = replaceArgumentMacros(result); //rescan for more macros +// popContext(); +// afterReplaceAllMacros(); +// } +// +// return arg; +// } +// +// +// private int expandFunctionStyleMacro(char[] expansion, +// CharArrayObjectMap argmap, CharArrayObjectMap replacedArgs, +// char[] result) { +// +// // The current position in the expansion string that we are looking at +// int pos = -1; +// // The last position in the expansion string that was copied over +// int lastcopy = -1; +// // The current write offset in the result string - also tells us the +// // length of the result string +// int outpos = 0; +// // The first character in the current block of white space - there are +// // times when we don't +// // want to copy over the whitespace +// int wsstart = -1; +// //whether or not we are on the second half of the ## operator +// boolean prevConcat = false; +// //for handling ## +// char[] prevArg = null; +// int prevArgStart = -1; +// int prevArgLength = -1; +// int prevArgTarget = 0; +// +// int limit = expansion.length; +// +// while (++pos < limit) { +// char c = expansion[pos]; +// +// if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' +// || (c >= '0' && c < '9') +// || Character.isUnicodeIdentifierPart(c)) { +// +// wsstart = -1; +// int idstart = pos; +// while (++pos < limit) { +// c = expansion[pos]; +// if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +// || (c >= '0' && c <= '9') || c == '_' || Character +// .isUnicodeIdentifierPart(c))) { +// break; +// } +// } +// --pos; +// +// char[] repObject = (char[]) argmap.get(expansion, idstart, pos +// - idstart + 1); +// +// int next = indexOfNextNonWhiteSpace(expansion, pos, limit); +// boolean nextIsPoundPound = (next + 1 < limit +// && expansion[next] == '#' && expansion[next + 1] == '#'); +// +// if (prevConcat && prevArgStart > -1 && prevArgLength > 0) { +// int l1 = prevArg != null ? prevArg.length : prevArgLength; +// int l2 = repObject != null ? repObject.length : pos +// - idstart + 1; +// char[] newRep = new char[l1 + l2]; +// if (prevArg != null) +// System.arraycopy(prevArg, 0, newRep, 0, l1); +// else +// System +// .arraycopy(expansion, prevArgStart, newRep, 0, +// l1); +// +// if (repObject != null) +// System.arraycopy(repObject, 0, newRep, l1, l2); +// else +// System.arraycopy(expansion, idstart, newRep, l1, l2); +// idstart = prevArgStart; +// repObject = newRep; +// } +// if (repObject != null) { +// // copy what we haven't so far +// if (++lastcopy < idstart) { +// int n = idstart - lastcopy; +// if (result != null) { +// // the outpos may be set back when prevConcat is true, so make sure we +// // stay in bounds. +// if (prevConcat && outpos+n > result.length) { +// n= result.length- outpos; +// } +// System.arraycopy(expansion, lastcopy, result, +// outpos, n); +// } +// outpos += n; +// } +// +// if (prevConcat) +// outpos = prevArgTarget; +// +// if (!nextIsPoundPound) { +// //16.3.1 completely macro replace the arguments before +// // substituting them in +// char[] rep = (char[]) ((replacedArgs != null) ? replacedArgs +// .get(repObject) +// : null); +// +// if (rep != null) +// repObject = rep; +// else { +// rep = replaceArgumentMacros(repObject); +// if (replacedArgs != null) +// replacedArgs.put(repObject, rep); +// repObject = rep; +// } +// +// if (result != null ) +// System.arraycopy(repObject, 0, result, outpos, repObject.length); +// } +// outpos += repObject.length; +// +// lastcopy = pos; +// } +// +// prevArg = repObject; +// prevArgStart = idstart; +// prevArgLength = pos - idstart + 1; +// prevArgTarget = repObject != null ? outpos - repObject.length +// : outpos + idstart - lastcopy - 1; +// prevConcat = false; +// } else if (c == '"') { +// +// // skip over strings +// wsstart = -1; +// boolean escaped = false; +// while (++pos < limit) { +// c = expansion[pos]; +// if (c == '"') { +// if (!escaped) +// break; +// } else if (c == '\\') { +// escaped = !escaped; +// } +// escaped = false; +// } +// prevConcat = false; +// } else if (c == '\'') { +// +// // skip over character literals +// wsstart = -1; +// boolean escaped = false; +// while (++pos < limit) { +// c = expansion[pos]; +// if (c == '\'') { +// if (!escaped) +// break; +// } else if (c == '\\') { +// escaped = !escaped; +// } +// escaped = false; +// } +// prevConcat = false; +// } else if (c == ' ' || c == '\t') { +// // obvious whitespace +// if (wsstart < 0) +// wsstart = pos; +// } else if (c == '/' && pos + 1 < limit) { +// +// // less than obvious, comments are whitespace +// c = expansion[pos+1]; +// if (c == '/') { +// // copy up to here or before the last whitespace +// ++pos; +// ++lastcopy; +// int n = wsstart < 0 ? pos - 1 - lastcopy : wsstart +// - lastcopy; +// if (result != null) +// System +// .arraycopy(expansion, lastcopy, result, outpos, +// n); +// outpos += n; +// +// // skip the rest +// lastcopy = expansion.length - 1; +// } else if (c == '*') { +// ++pos; +// if (wsstart < 1) +// wsstart = pos - 1; +// while (++pos < limit) { +// if (expansion[pos] == '*' && pos + 1 < limit +// && expansion[pos + 1] == '/') { +// ++pos; +// break; +// } +// } +// } else +// wsstart = -1; +// +// } else if (c == '\\' && pos + 1 < limit +// && expansion[pos + 1] == 'n') { +// // skip over this +// ++pos; +// +// } else if (c == '#') { +// +// if (pos + 1 < limit && expansion[pos + 1] == '#') { +// prevConcat = true; +// ++pos; +// // skip whitespace +// if (wsstart < 0) +// wsstart = pos - 1; +// while (++pos < limit) { +// switch (expansion[pos]) { +// case ' ': +// case '\t': +// continue; +// +// case '/': +// if (pos + 1 < limit) { +// c = expansion[pos + 1]; +// if (c == '/') +// // skip over everything +// pos = expansion.length; +// else if (c == '*') { +// ++pos; +// while (++pos < limit) { +// if (expansion[pos] == '*' +// && pos + 1 < limit +// && expansion[pos + 1] == '/') { +// ++pos; +// break; +// } +// } +// continue; +// } +// } +// } +// break; +// } +// --pos; +// } else { +// prevConcat = false; +// // stringify +// +// // copy what we haven't so far +// if (++lastcopy < pos) { +// int n = pos - lastcopy; +// if (result != null) +// System.arraycopy(expansion, lastcopy, result, +// outpos, n); +// outpos += n; +// } +// +// // skip whitespace +// while (++pos < limit) { +// switch (expansion[pos]) { +// case ' ': +// case '\t': +// continue; +// case '/': +// if (pos + 1 < limit) { +// c = expansion[pos + 1]; +// if (c == '/') +// // skip over everything +// pos = expansion.length; +// else if (c == '*') { +// ++pos; +// while (++pos < limit) { +// if (expansion[pos] == '*' +// && pos + 1 < limit +// && expansion[pos + 1] == '/') { +// ++pos; +// break; +// } +// } +// continue; +// } +// } +// //TODO handle comments +// } +// break; +// } +// +// // grab the identifier +// c = expansion[pos]; +// int idstart = pos; +// if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'X') +// || c == '_' || Character.isUnicodeIdentifierPart(c)) { +// while (++pos < limit) { +// c = expansion[pos]; +// if (!((c >= 'a' && c <= 'z') +// || (c >= 'A' && c <= 'X') +// || (c >= '0' && c <= '9') || c == '_' || Character +// .isUnicodeIdentifierPart(c))) +// break; +// } +// } // else TODO something +// --pos; +// int idlen = pos - idstart + 1; +// char[] argvalue = (char[]) argmap.get(expansion, idstart, +// idlen); +// if (argvalue != null) { +// //16.3.2-2 ... a \ character is inserted before each " +// // and \ +// // character +// //of a character literal or string literal +// +// //technically, we are also supposed to replace each +// // occurence +// // of whitespace +// //(including comments) in the argument with a single +// // space. +// // But, at this time +// //we don't really care what the contents of the string +// // are, +// // just that we get the string +// //so we won't bother doing that +// if (result != null) { +// result[outpos++] = '"'; +// for (int i = 0; i < argvalue.length; i++) { +// if (argvalue[i] == '"' || argvalue[i] == '\\') +// result[outpos++] = '\\'; +// if (argvalue[i] == '\r' || argvalue[i] == '\n') +// result[outpos++] = ' '; +// else +// result[outpos++] = argvalue[i]; +// } +// result[outpos++] = '"'; +// } else { +// for (int i = 0; i < argvalue.length; i++) { +// if (argvalue[i] == '"' || argvalue[i] == '\\') +// ++outpos; +// ++outpos; +// } +// outpos += 2; +// } +// } +// lastcopy = pos; +// wsstart = -1; +// } +// } else { +// prevConcat = false; +// // not sure what it is but it sure ain't whitespace +// wsstart = -1; +// } +// +// } +// +// if (wsstart < 0 && ++lastcopy < expansion.length) { +// int n = expansion.length - lastcopy; +// if (result != null) +// System.arraycopy(expansion, lastcopy, result, outpos, n); +// outpos += n; +// } +// +// return outpos; +// } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java new file mode 100644 index 00000000000..3f58c8a7682 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java @@ -0,0 +1,375 @@ +/******************************************************************************* + * Copyright (c) 2004, 2007 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM Corporation - initial implementation + * Markus Schorn (Wind River Systems) + * Bryan Wilkinson (QNX) - https://bugs.eclipse.org/bugs/show_bug.cgi?id=151207 + * Anton Leherbauer (Wind River Systems) + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import org.eclipse.cdt.core.parser.IProblem; +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.util.CharArrayObjectMap; + +/** + * Used to evaluate expressions in preprocessor directives. + * @since 5.0 + */ +class ExpressionEvaluator { + static class EvalException extends Exception { + private int fProblemID; + private char[] fProblemArg; + + public EvalException(int problemID, char[] problemArg) { + fProblemID= problemID; + } + + public int getProblemID() { + return fProblemID; + } + + public char[] getProblemArg() { + return fProblemArg; + } + } + + private Token fTokens; + private CharArrayObjectMap fDictionary; + + public boolean evaluate(Token condition, CharArrayObjectMap dictionary) throws EvalException { + fTokens= condition; + fDictionary= dictionary; + return expression() != 0; + } + + + private long expression() throws EvalException { + return conditionalExpression(); + } + + private long conditionalExpression() throws EvalException { + long r1 = logicalOrExpression(); + if (LA() == IToken.tQUESTION) { + consume(); + long r2 = expression(); + if (LA() == IToken.tCOLON) + consume(); + else { + throw new EvalException(IProblem.SCANNER_BAD_CONDITIONAL_EXPRESSION, null); + } + long r3 = conditionalExpression(); + return r1 != 0 ? r2 : r3; + } + return r1; + } + + private long logicalOrExpression() throws EvalException { + long r1 = logicalAndExpression(); + while (LA() == IToken.tOR) { + consume(); + long r2 = logicalAndExpression(); + r1 = ((r1 != 0) || (r2 != 0)) ? 1 : 0; + } + return r1; + } + + private long logicalAndExpression() throws EvalException { + long r1 = inclusiveOrExpression(); + while (LA() == IToken.tAND) { + consume(); + long r2 = inclusiveOrExpression(); + r1 = ((r1 != 0) && (r2 != 0)) ? 1 : 0; + } + return r1; + } + + private long inclusiveOrExpression() throws EvalException { + long r1 = exclusiveOrExpression(); + while (LA() == IToken.tBITOR) { + consume(); + long r2 = exclusiveOrExpression(); + r1 = r1 | r2; + } + return r1; + } + + private long exclusiveOrExpression() throws EvalException { + long r1 = andExpression(); + while (LA() == IToken.tXOR) { + consume(); + long r2 = andExpression(); + r1 = r1 ^ r2; + } + return r1; + } + + private long andExpression() throws EvalException { + long r1 = equalityExpression(); + while (LA() == IToken.tAMPER) { + consume(); + long r2 = equalityExpression(); + r1 = r1 & r2; + } + return r1; + } + + private long equalityExpression() throws EvalException { + long r1 = relationalExpression(); + for (int t = LA(); t == IToken.tEQUAL || t == IToken.tNOTEQUAL; t = LA()) { + consume(); + long r2 = relationalExpression(); + if (t == IToken.tEQUAL) + r1 = (r1 == r2) ? 1 : 0; + else + // t == tNOTEQUAL + r1 = (r1 != r2) ? 1 : 0; + } + return r1; + } + + private long relationalExpression() throws EvalException { + long r1 = shiftExpression(); + for (int t = LA(); t == IToken.tLT || t == IToken.tLTEQUAL || t == IToken.tGT + || t == IToken.tGTEQUAL; t = LA()) { + consume(); + long r2 = shiftExpression(); + switch (t) { + case IToken.tLT: + r1 = (r1 < r2) ? 1 : 0; + break; + case IToken.tLTEQUAL: + r1 = (r1 <= r2) ? 1 : 0; + break; + case IToken.tGT: + r1 = (r1 > r2) ? 1 : 0; + break; + case IToken.tGTEQUAL: + r1 = (r1 >= r2) ? 1 : 0; + break; + } + } + return r1; + } + + private long shiftExpression() throws EvalException { + long r1 = additiveExpression(); + for (int t = LA(); t == IToken.tSHIFTL || t == IToken.tSHIFTR; t = LA()) { + consume(); + long r2 = additiveExpression(); + if (t == IToken.tSHIFTL) + r1 = r1 << r2; + else + // t == tSHIFTR + r1 = r1 >> r2; + } + return r1; + } + + private long additiveExpression() throws EvalException { + long r1 = multiplicativeExpression(); + for (int t = LA(); t == IToken.tPLUS || t == IToken.tMINUS; t = LA()) { + consume(); + long r2 = multiplicativeExpression(); + if (t == IToken.tPLUS) + r1 = r1 + r2; + else + // t == tMINUS + r1 = r1 - r2; + } + return r1; + } + + private long multiplicativeExpression() throws EvalException { + long r1 = unaryExpression(); + for (int t = LA(); t == IToken.tSTAR || t == IToken.tDIV || t == IToken.tMOD; t = LA()) { + consume(); + long r2 = unaryExpression(); + if (t == IToken.tSTAR) + r1 = r1 * r2; + else if (r2 != 0) { + if (t == IToken.tDIV) + r1 = r1 / r2; + else + r1 = r1 % r2; //tMOD + } else { + throw new EvalException(IProblem.SCANNER_DIVIDE_BY_ZERO, null); + } + } + return r1; + } + + private long unaryExpression() throws EvalException { + switch (LA()) { + case IToken.tPLUS: + consume(); + return unaryExpression(); + case IToken.tMINUS: + consume(); + return -unaryExpression(); + case IToken.tNOT: + consume(); + return unaryExpression() == 0 ? 1 : 0; + case IToken.tBITCOMPLEMENT: + consume(); + return ~unaryExpression(); + case IToken.tCHAR: + case IToken.tLCHAR: + case IToken.tINTEGER: + long val= getValue(fTokens); + consume(); + return val; + case CPreprocessor.tDEFINED: + return handleDefined(); + case IToken.tLPAREN: + consume(); + long r1 = expression(); + if (LA() == IToken.tRPAREN) { + consume(); + return r1; + } + throw new EvalException(IProblem.SCANNER_MISSING_R_PAREN, null); + case IToken.tIDENTIFIER: + return 1; + default: + throw new EvalException(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, null); + } + } + + private long handleDefined() throws EvalException { + boolean parenthesis= false; + consume(); + if (LA() == IToken.tLPAREN) { + parenthesis= true; + consume(); + } + if (LA() != IToken.tIDENTIFIER) { + throw new EvalException(IProblem.SCANNER_ILLEGAL_IDENTIFIER, null); + } + int result= fDictionary.containsKey(fTokens.getCharImage()) ? 1 : 0; + consume(); + if (parenthesis) { + if (LA() != IToken.tRPAREN) { + throw new EvalException(IProblem.SCANNER_MISSING_R_PAREN, null); + } + consume(); + } + return result; + } + + private int LA() { + return fTokens.getType(); + } + + private void consume() { + fTokens= (Token) fTokens.getNext(); + if (fTokens == null) { + fTokens= new SimpleToken(Lexer.tEND_OF_INPUT, 0, 0); + } + } + + long getValue(Token t) throws EvalException { + switch(t.getType()) { + case IToken.tCHAR: + return getChar(t.getCharImage(), 1); + case IToken.tLCHAR: + return getChar(t.getCharImage(), 2); + case IToken.tINTEGER: + return getNumber(t.getCharImage()); + } + assert false; + return 1; + } + + private long getNumber(char[] image) throws EvalException { + boolean isHex = false; + boolean isOctal = false; + + int pos= 0; + if (image.length > 1) { + if (image[0] == '0') { + switch (image[++pos]) { + case 'x': + case 'X': + isHex = true; + ++pos; + break; + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + isOctal = true; + ++pos; + break; + } + } + } + if (isHex) { + return getNumber(image, 2, image.length, 16, IProblem.SCANNER_BAD_HEX_FORMAT); + } + if (isOctal) { + return getNumber(image, 1, image.length, 8, IProblem.SCANNER_BAD_OCTAL_FORMAT); + } + return getNumber(image, 0, image.length, 10, IProblem.SCANNER_BAD_DECIMAL_FORMAT); + } + + private long getChar(char[] tokenImage, int i) throws EvalException { + if (i>=tokenImage.length) { + throw new EvalException(IProblem.SCANNER_BAD_CHARACTER, tokenImage); + } + final char c= tokenImage[i]; + if (c != '\\') { + return c; + } + + if (++i>=tokenImage.length) { + throw new EvalException(IProblem.SCANNER_BAD_CHARACTER, tokenImage); + } + final char d= tokenImage[i]; + switch(d) { + case '\\': case '"': case '\'': + return d; + case 'a': return 7; + case 'b': return '\b'; + case 'f': return '\f'; + case 'n': return '\n'; + case 'r': return '\r'; + case 't': return '\t'; + case 'v': return 0xb; + + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': + return getNumber(tokenImage, i+1, tokenImage.length-1, 8, IProblem.SCANNER_BAD_OCTAL_FORMAT); + + case 'x': case 'u': case 'U': + return getNumber(tokenImage, i+1, tokenImage.length-1, 16, IProblem.SCANNER_BAD_HEX_FORMAT); + default: + throw new EvalException(IProblem.SCANNER_BAD_CHARACTER, tokenImage); + } + } + + private long getNumber(char[] tokenImage, int from, int to, int base, int problemID) throws EvalException { + long result= 0; + for (int i = from; i < to; i++) { + int digit= getDigit(tokenImage[i]); + if (digit >= base) { + throw new EvalException(problemID, tokenImage); + } + result= result*base + digit; + } + return result; + } + + private int getDigit(char c) { + switch(c) { + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + return c-'0'; + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': + return c-'a' + 10; + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': + return c-'A'+10; + } + return Integer.MAX_VALUE; + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILexerLog.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILexerLog.java index 8c5c9043b5a..53243a31cd6 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILexerLog.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILexerLog.java @@ -10,9 +10,29 @@ *******************************************************************************/ package org.eclipse.cdt.internal.core.parser.scanner; +import org.eclipse.cdt.core.parser.IProblem; + +/** + * Interface between the lexer and the preprocessor for picking up warnings and comments. + * @since 5.0 + */ public interface ILexerLog { - void handleProblem(int problemID, char[] source, int offset, int endOffset); + /** + * A problem has been detected + * @param problemID id as defined in {@link IProblem} + * @param info additional info as required for {@link IProblem}. + * @param offset The offset of the problem in the source of the lexer. + * @param endOffset end offset of the problem in the source of the lexer. + */ + void handleProblem(int problemID, char[] info, int offset, int endOffset); - void handleComment(boolean isBlockComment, char[] source, int offset, int endOffsetLast); + /** + * A comment has been detected + * @param isBlockComment true for block-comments, false for line-comments. + * @param source the input of the lexer. + * @param offset the offset where the comment starts + * @param endOffset the offset where the comment ends + */ + void handleComment(boolean isBlockComment, int offset, int endOffset); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CompletionTokenException.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationCtx.java similarity index 73% rename from core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CompletionTokenException.java rename to core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationCtx.java index 506f7898ee2..22931903bff 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CompletionTokenException.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationCtx.java @@ -8,17 +8,14 @@ * Contributors: * Markus Schorn - initial API and implementation *******************************************************************************/ + package org.eclipse.cdt.internal.core.parser.scanner; -class CompletionTokenException extends Exception { +/** + * Interface for modeling contexts that can deal with offsets. These are: + * synthetic contexts used for pre-included files, file-contexts, macro-expansions. + * @since 5.0 + */ +public interface ILocationCtx { - private Token fToken; - - public CompletionTokenException(Token token) { - fToken= token; - } - - public Token getToken() { - return fToken; - } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationResolver.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationResolver.java new file mode 100644 index 00000000000..04f86ed453f --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ILocationResolver.java @@ -0,0 +1,85 @@ +/******************************************************************************* + * Copyright (c) 2004, 2007 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM - Initial API and implementation + * Markus Schorn (Wind River Systems) + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import org.eclipse.cdt.core.dom.ast.IASTName; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIncludeStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorMacroDefinition; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorStatement; +import org.eclipse.cdt.core.dom.ast.IASTProblem; +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit; +import org.eclipse.cdt.core.dom.ast.IMacroBinding; +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit.IDependencyTree; + + +/** + * Interface between the ast and the location-resolver for resolving offsets. + * @since 5.0 + */ +public interface ILocationResolver extends org.eclipse.cdt.internal.core.parser.scanner2.ILocationResolver { + + /** + * Introduces the ast translation unit to the location resolver. Must be called before any tokens from the + * scanner are obtained. + */ + void setRootNode(IASTTranslationUnit tu); + + /** + * @see IASTTranslationUnit#getAllPreprocessorStatements() + */ + IASTPreprocessorStatement [] getAllPreprocessorStatements(); + + /** + * @see IASTTranslationUnit#getMacroDefinitions() + */ + IASTPreprocessorMacroDefinition [] getMacroDefinitions(); + + /** + * @see IASTTranslationUnit#getBuiltinMacroDefinitions() + */ + IASTPreprocessorMacroDefinition [] getBuiltinMacroDefinitions(); + + /** + * @see IASTTranslationUnit#getIncludeDirectives() + */ + IASTPreprocessorIncludeStatement [] getIncludeDirectives(); + + /** + * @see IASTTranslationUnit#getPreprocessorProblems() + */ + IASTProblem[] getScannerProblems(); + + /** + * @see IASTTranslationUnit#getFilePath() + */ + public String getTranslationUnitPath(); + + /** + * @see IASTTranslationUnit#getContainingFilename() + */ + public String getContainingFilename(int offset); + + /** + * @see IASTTranslationUnit#getDependencyTree() + */ + public IDependencyTree getDependencyTree(); + + /** + * Returns explicit and implicit references for a macro. + */ + public IASTName[] getReferences(IMacroBinding binding); + + /** + * Returns the definition for a macro. + */ + public IASTName[] getDeclarations(IMacroBinding binding); +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/DigraphToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IPreprocessorMacro.java similarity index 72% rename from core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/DigraphToken.java rename to core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IPreprocessorMacro.java index d83c81e8c14..4f0e24f78de 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/DigraphToken.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IPreprocessorMacro.java @@ -10,12 +10,12 @@ *******************************************************************************/ package org.eclipse.cdt.internal.core.parser.scanner; -class DigraphToken extends Token { - public DigraphToken(int kind, int offset, int endOffset) { - super(kind, offset, endOffset); - } +import org.eclipse.cdt.core.dom.ast.IMacroBinding; + +/** + * Interface for the location map when using the macros from the preprocessor. + * @since 5.0 + */ +public interface IPreprocessorMacro extends ILocationCtx, IMacroBinding { - public char[] getTokenImage() { - return TokenUtil.getDigraphImage(getType()); - } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/SimpleToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ImageLocationInfo.java similarity index 68% rename from core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/SimpleToken.java rename to core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ImageLocationInfo.java index d930f044a16..980ed341097 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/SimpleToken.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ImageLocationInfo.java @@ -10,12 +10,12 @@ *******************************************************************************/ package org.eclipse.cdt.internal.core.parser.scanner; -class SimpleToken extends Token { - public SimpleToken(int kind, int offset, int endOffset) { - super(kind, offset, endOffset); - } +/** + * Information needed for computing image-locations. An image location exists for a name and describes where the name + * came from. This can be: source code, macro-expansion, parameter to macro-expansion or synthetic. + * + * @since 5.0 + */ +class ImageLocationInfo { - public char[] getTokenImage() { - return TokenUtil.getImage(getType()); - } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java index eabddb459b8..2252135b303 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java @@ -14,6 +14,7 @@ import org.eclipse.cdt.core.dom.ast.IASTProblem; import org.eclipse.cdt.core.parser.IGCCToken; import org.eclipse.cdt.core.parser.IProblem; import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.OffsetLimitReachedException; /** * In short this class converts line endings (to '\n') and trigraphs @@ -36,6 +37,7 @@ import org.eclipse.cdt.core.parser.IToken; * an execution character-set is performed. */ final public class Lexer { + public static final int tBEFORE_INPUT = IToken.FIRST_RESERVED_SCANNER; public static final int tNEWLINE = IToken.FIRST_RESERVED_SCANNER + 1; public static final int tEND_OF_INPUT = IToken.FIRST_RESERVED_SCANNER + 2; public static final int tQUOTE_HEADER_NAME = IToken.FIRST_RESERVED_SCANNER + 3; @@ -43,11 +45,20 @@ final public class Lexer { private static final int END_OF_INPUT = -1; private static final int LINE_SPLICE_SEQUENCE = -2; + private static final int ORIGIN_LEXER = OffsetLimitReachedException.ORIGIN_LEXER; - public static class LexerOptions { + public final static class LexerOptions implements Cloneable { public boolean fSupportDollarInitializers= true; public boolean fSupportMinAndMax= true; public boolean fSupportContentAssist= false; + + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException e) { + return null; + } + } } // configuration @@ -56,7 +67,7 @@ final public class Lexer { // the input to the lexer private final char[] fInput; - private final int fLimit; + private int fLimit; // after phase 3 (newline, trigraph, line-splice) private int fOffset; @@ -64,12 +75,13 @@ final public class Lexer { private int fCharPhase3; private boolean fInsideIncludeDirective= false; - private Token fToken; + private Token fToken= new SimpleToken(tBEFORE_INPUT, 0, 0); // for the few cases where we have to lookahead more than one character private int fMarkOffset; private int fMarkEndOffset; private int fMarkPrefetchedChar; + private boolean fFirstTokenAfterNewline= true; public Lexer(char[] input, LexerOptions options, ILexerLog log) { @@ -88,6 +100,17 @@ final public class Lexer { nextCharPhase3(); } + /** + * Resets the lexer to the first char and prepares for content-assist mode. + */ + public void setContentAssistMode(int offset) { + fOptions.fSupportContentAssist= true; + fLimit= Math.min(fLimit, fInput.length); + // re-initialize + fOffset= fEndOffset= 0; + nextCharPhase3(); + } + /** * Call this before consuming the name-token in the include directive. It causes the header-file * tokens to be created. @@ -105,18 +128,53 @@ final public class Lexer { /** * Advances to the next token, skipping whitespace other than newline. - * @throws CompletionTokenException when completion is requested in a literal or an header-name. + * @throws OffsetLimitReachedException when completion is requested in a literal or a header-name. */ - public Token nextToken() throws CompletionTokenException { + public Token nextToken() throws OffsetLimitReachedException { + fFirstTokenAfterNewline= fToken.getType() == tNEWLINE; return fToken= fetchToken(); } + public boolean currentTokenIsFirstOnLine() { + return fFirstTokenAfterNewline; + } + + /** + * Advances to the next newline. + * @return the end offset of the last token before the newline or the start of the newline + * if there were no other tokens. + * @param origin parameter for the {@link OffsetLimitReachedException} when it has to be thrown. + * @since 5.0 + */ + public final int consumeLine(int origin) throws OffsetLimitReachedException { + Token t= fToken; + Token lt= null; + while(true) { + switch(t.getType()) { + case IToken.tCOMPLETION: + fToken= t; + throw new OffsetLimitReachedException(origin, t); + case Lexer.tEND_OF_INPUT: + fToken= t; + if (fOptions.fSupportContentAssist) { + throw new OffsetLimitReachedException(origin, lt); + } + return lt != null ? lt.getEndOffset() : t.getOffset(); + case Lexer.tNEWLINE: + fToken= t; + return lt != null ? lt.getEndOffset() : t.getOffset(); + } + lt= t; + t= fetchToken(); + } + } + /** * Advances to the next pound token that starts a preprocessor directive. * @return pound token of the directive or end-of-input. - * @throws CompletionTokenException when completion is requested in a literal or an header-name. + * @throws OffsetLimitReachedException when completion is requested in a literal or an header-name. */ - public Token nextDirective() throws CompletionTokenException { + public Token nextDirective() throws OffsetLimitReachedException { Token t= fToken; boolean haveNL= t==null || t.getType() == tNEWLINE; loop: while(true) { @@ -147,7 +205,7 @@ final public class Lexer { /** * Computes the next token. */ - private Token fetchToken() throws CompletionTokenException { + private Token fetchToken() throws OffsetLimitReachedException { while(true) { final int start= fOffset; final int c= fCharPhase3; @@ -454,7 +512,7 @@ final public class Lexer { break; } - handleProblem(IASTProblem.SCANNER_BAD_CHARACTER, start); + handleProblem(IASTProblem.SCANNER_BAD_CHARACTER, new char[] {(char) c}, start); // loop is continued, character is treated as white-space. } } @@ -467,15 +525,20 @@ final public class Lexer { return new DigraphToken(kind, offset, fOffset); } - private Token newToken(int kind, int offset, int length) { - return new TokenWithImage(kind, this, offset, fOffset, length); + private Token newToken(int kind, int offset, int imageLength) { + final int endOffset= fOffset; + int sourceLen= endOffset-offset; + if (sourceLen != imageLength) { + return new ImageToken(kind, offset, endOffset, getCharImage(offset, endOffset, imageLength)); + } + return new SourceImageToken(kind, offset, endOffset, fInput); } - private void handleProblem(int problemID, int offset) { - fLog.handleProblem(problemID, fInput, offset, fOffset); + private void handleProblem(int problemID, char[] arg, int offset) { + fLog.handleProblem(problemID, arg, offset, fOffset); } - private Token headerName(final int start, final boolean expectQuotes) throws CompletionTokenException { + private Token headerName(final int start, final boolean expectQuotes) throws OffsetLimitReachedException { int length= 1; boolean done = false; int c= fCharPhase3; @@ -483,12 +546,12 @@ final public class Lexer { switch (c) { case END_OF_INPUT: if (fOptions.fSupportContentAssist) { - throw new CompletionTokenException( + throw new OffsetLimitReachedException(ORIGIN_LEXER, newToken((expectQuotes ? tQUOTE_HEADER_NAME : tSYSTEM_HEADER_NAME), start, length)); } // no break; case '\n': - handleProblem(IProblem.SCANNER_UNBOUNDED_STRING, start); + handleProblem(IProblem.SCANNER_UNBOUNDED_STRING, getInputChars(start, fOffset), start); break loop; case '"': @@ -509,13 +572,13 @@ final public class Lexer { while(true) { switch (c) { case END_OF_INPUT: - fLog.handleComment(true, fInput, start, fOffset); + fLog.handleComment(true, start, fOffset); return; case '*': c= nextCharPhase3(); if (c == '/') { nextCharPhase3(); - fLog.handleComment(true, fInput, start, fOffset); + fLog.handleComment(true, start, fOffset); return; } break; @@ -532,14 +595,14 @@ final public class Lexer { switch (c) { case END_OF_INPUT: case '\n': - fLog.handleComment(false, fInput, start, fOffset); + fLog.handleComment(false, start, fOffset); return; } c= nextCharPhase3(); } } - private Token stringLiteral(final int start, final boolean wide) throws CompletionTokenException { + private Token stringLiteral(final int start, final boolean wide) throws OffsetLimitReachedException { boolean escaped = false; boolean done = false; int length= wide ? 2 : 1; @@ -549,11 +612,11 @@ final public class Lexer { switch(c) { case END_OF_INPUT: if (fOptions.fSupportContentAssist) { - throw new CompletionTokenException(newToken(wide ? IToken.tLSTRING : IToken.tSTRING, start, length)); + throw new OffsetLimitReachedException(ORIGIN_LEXER, newToken(wide ? IToken.tLSTRING : IToken.tSTRING, start, length)); } // no break; case '\n': - handleProblem(IProblem.SCANNER_UNBOUNDED_STRING, start); + handleProblem(IProblem.SCANNER_UNBOUNDED_STRING, getInputChars(start, fOffset), start); break loop; case '\\': @@ -575,7 +638,7 @@ final public class Lexer { return newToken(wide ? IToken.tLSTRING : IToken.tSTRING, start, length); } - private Token charLiteral(final int start, boolean wide) throws CompletionTokenException { + private Token charLiteral(final int start, boolean wide) throws OffsetLimitReachedException { boolean escaped = false; boolean done = false; int length= wide ? 2 : 1; @@ -585,11 +648,11 @@ final public class Lexer { switch(c) { case END_OF_INPUT: if (fOptions.fSupportContentAssist) { - throw new CompletionTokenException(newToken(wide ? IToken.tLCHAR : IToken.tCHAR, start, length)); + throw new OffsetLimitReachedException(ORIGIN_LEXER, newToken(wide ? IToken.tLCHAR : IToken.tCHAR, start, length)); } // no break; case '\n': - handleProblem(IProblem.SCANNER_BAD_CHARACTER, start); + handleProblem(IProblem.SCANNER_BAD_CHARACTER, getInputChars(start, fOffset), start); break loop; case '\\': escaped= !escaped; @@ -676,7 +739,7 @@ final public class Lexer { return newToken(tokenKind, start, length); } - private Token number(final int start, int length, boolean isFloat) throws CompletionTokenException { + private Token number(final int start, int length, boolean isFloat) throws OffsetLimitReachedException { boolean isPartOfNumber= true; int c= fCharPhase3; while (true) { @@ -733,7 +796,7 @@ final public class Lexer { case tEND_OF_INPUT: if (fOptions.fSupportContentAssist) { - throw new CompletionTokenException( + throw new OffsetLimitReachedException(ORIGIN_LEXER, newToken((isFloat ? IToken.tFLOATINGPT : IToken.tINTEGER), start, length)); } isPartOfNumber= false; @@ -916,27 +979,25 @@ final public class Lexer { return result; } + char[] getInput() { + return fInput; + } + /** * Returns the image with trigraphs replaced and line-splices removed. */ - char[] getTokenImage(int offset, int endOffset, int imageLength) { - final int length= endOffset-offset; + private char[] getCharImage(int offset, int endOffset, int imageLength) { final char[] result= new char[imageLength]; - if (length == imageLength) { - System.arraycopy(fInput, offset, result, 0, length); - } - else { - markPhase3(); - fEndOffset= offset; - int idx= 0; - while (idxtrue. + */ + public int getSequenceNumberForOffset(int offset, boolean checkChildren) { + return fSequenceNumber+offset; + } + + /** + * When a child-context is finished it reports its total sequence length, such that offsets in this + * context can be converted to sequence numbers. + */ + public void addChildSequenceLength(int childLength) { + assert false; + } + + /** + * Returns the line number for an offset within this context. Not all contexts support line numbers, + * so this may return 0. + */ + public int getLineNumber(int offset) { + return 0; + } + + /** + * Returns the minimal context containing the specified range, assuming that it is contained in + * this context. + */ + public LocationCtx findContextForSequenceNumberRange(int sequenceNumber, int length) { + return this; + } + + /** + * Returns the minimal file location containing the specified sequence number range, assuming + * that it is contained in this context. + */ + public IASTFileLocation getFileLocationForSequenceNumberRange(int sequenceNumber, int length) { + return fParent.getFileLocationForOffsetRange(fParentOffset, fParentEndOffset-fParentOffset); + } + + /** + * Returns the file location containing the specified offset range in this context. + */ + public IASTFileLocation getFileLocationForOffsetRange(int parentOffset, int length) { + return fParent.getFileLocationForOffsetRange(fParentOffset, fParentEndOffset-fParentOffset); + } + + /** + * Support for the dependency tree, add inclusion statements found in this context. + */ + public void getInclusions(ArrayList target) { + } + + /** + * Support for the dependency tree, returns inclusion statement that created this context, or null. + */ + public ASTInclusionStatement getInclusionStatement() { + return null; + } +} + +class ContainerLocationCtx extends LocationCtx { + private int fChildSequenceLength; + private ArrayList fChildren; + private char[] fSource; + + public ContainerLocationCtx(LocationCtx parent, char[] source, int parentOffset, int parentEndOffset, int sequenceNumber) { + super(parent, parentOffset, parentEndOffset, sequenceNumber); + fSource= source; + } + + public final int getSequenceLength() { + return fSource.length + fChildSequenceLength; + } + public final int getSequenceNumberForOffset(int offset, boolean checkChildren) { + int result= fSequenceNumber + fChildSequenceLength + offset; + if (checkChildren && fChildren != null) { + for (int i= fChildren.size()-1; i >= 0; i--) { + final LocationCtx child= (LocationCtx) fChildren.get(i); + if (child.fParentEndOffset > offset) { // child was inserted behind the offset, adjust sequence number + result-= child.getSequenceLength(); + } + else { + return result; + } + } + } + return result; + } + + public void addChildSequenceLength(int childLength) { + fChildSequenceLength+= childLength; + } + + public final LocationCtx findContextForSequenceNumberRange(int sequenceNumber, int length) { + final LocationCtx child= findChildLessOrEqualThan(sequenceNumber); + if (child != null && child.fSequenceNumber+child.getSequenceLength() >= sequenceNumber+length) { + return child; + } + return this; + } + + public IASTFileLocation getFileLocationForSequenceNumberRange(int sequenceNumber, int length) { + // try to delegate to a child. + int useLength= length > 0 ? length-1 : 0; + final LocationCtx child1= findChildLessOrEqualThan(sequenceNumber); + final LocationCtx child2= findChildLessOrEqualThan(sequenceNumber+useLength); + if (child1 == child2 && child1 != null) { + return child1.getFileLocationForOffsetRange(sequenceNumber, length); + } + return super.getFileLocationForSequenceNumberRange(sequenceNumber, length); + } + + final LocationCtx findChildLessOrEqualThan(final int sequenceNumber) { + if (fChildren == null) { + return null; + } + int upper= fChildren.size(); + if (upper < 10) { + for (int i=upper-1; i>=0; i--) { + LocationCtx child= (LocationCtx) fChildren.get(i); + if (child.fSequenceNumber <= sequenceNumber) { + return child; + } + } + return null; + } + + int lower= 0; + while (upper > lower) { + int middle= (upper+lower)/2; + LocationCtx child= (LocationCtx) fChildren.get(middle); + if (child.fSequenceNumber <= sequenceNumber) { + lower= middle+1; + } + else { + upper= middle; + } + } + if (lower > 0) { + return (LocationCtx) fChildren.get(lower-1); + } + return null; + } + + public void getInclusions(ArrayList result) { + for (Iterator iterator = fChildren.iterator(); iterator.hasNext();) { + LocationCtx ctx= (LocationCtx) iterator.next(); + if (ctx.getInclusionStatement() != null) { + result.add(new ASTInclusionNode(ctx)); + } + else { + ctx.getInclusions(result); + } + } + } +} + +class FileLocationCtx extends ContainerLocationCtx { + private final String fFilename; + private final ASTInclusionStatement fASTInclude; + + public FileLocationCtx(LocationCtx parent, String filename, char[] source, int parentOffset, int parentEndOffset, int sequenceNumber, ASTInclusionStatement inclusionStatement) { + super(parent, source, parentOffset, parentEndOffset, sequenceNumber); + fFilename= new String(filename); + fASTInclude= inclusionStatement; + } + + public final void addChildSequenceLength(int childLength) { + super.addChildSequenceLength(childLength); + if (fASTInclude != null) { + fASTInclude.setLength(fASTInclude.getLength()+childLength); + } + } + + public final String getFilename() { + return fFilename; + } + + public IASTFileLocation getFileLocationForSequenceNumberRange(int sequenceNumber, int length) { + // try to delegate to a child. + final int sequenceEnd= sequenceNumber+length; + final LocationCtx child1= findChildLessOrEqualThan(sequenceNumber); + final LocationCtx child2= sequenceEnd == sequenceNumber ? child1 : findChildLessOrEqualThan(sequenceEnd-1); + if (child1 == child2 && child1 != null) { + return child1.getFileLocationForOffsetRange(sequenceNumber, length); + } + + // handle here + int startOffset; + int endOffset; + + if (child1 == null) { + startOffset= sequenceNumber-fSequenceNumber; + } + else { + int childSequenceEnd= child1.fSequenceNumber + child1.getSequenceLength(); + if (sequenceNumber < childSequenceEnd) { + startOffset= child1.fParentOffset; + } + else { // start beyond child1 + startOffset= child1.fParentEndOffset + sequenceNumber-childSequenceEnd; + } + } + if (child2 == null) { + endOffset= sequenceEnd-fSequenceNumber; + } + else { + int childSequenceEnd= child2.fSequenceNumber + child2.getSequenceLength(); + if (childSequenceEnd < sequenceEnd) { // beyond child2 + endOffset= child2.fParentEndOffset+sequenceEnd-childSequenceEnd; + } + else { + endOffset= child2.fParentEndOffset; + } + } + return new ASTFileLocation(fFilename, startOffset, endOffset-startOffset); + } + + public int getLineNumber(int offset) { + // mstodo Auto-generated method stub + return super.getLineNumber(offset); + } + + public ASTInclusionStatement getInclusionStatement() { + return fASTInclude; + } +} + + +class MacroExpansionCtx extends LocationCtx { + private final int fLength; + + public MacroExpansionCtx(LocationCtx parent, int parentOffset, int parentEndOffset, + int sequenceNumber, int length, ImageLocationInfo[] imageLocations, ASTPreprocessorName expansion) { + super(parent, parentOffset, parentEndOffset, sequenceNumber); + fLength= length; + } + + public int getSequenceLength() { + return fLength; + } + + // mstodo once image locations are supported we need to handle those in here +} \ No newline at end of file diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LocationMap.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LocationMap.java new file mode 100644 index 00000000000..c161d19ce1f --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LocationMap.java @@ -0,0 +1,432 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - Initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.List; + +import org.eclipse.cdt.core.dom.ast.IASTFileLocation; +import org.eclipse.cdt.core.dom.ast.IASTName; +import org.eclipse.cdt.core.dom.ast.IASTNodeLocation; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIncludeStatement; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorMacroDefinition; +import org.eclipse.cdt.core.dom.ast.IASTPreprocessorStatement; +import org.eclipse.cdt.core.dom.ast.IASTProblem; +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit; +import org.eclipse.cdt.core.dom.ast.IMacroBinding; +import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit.IDependencyTree; +import org.eclipse.cdt.internal.core.dom.parser.ASTPreprocessorSelectionResult; + +/** + * Converts the offsets relative to various contexts to the global sequence number. Also creates and stores + * objects that are needed to conform with the IAST... interfaces. + * @since 5.0 + */ +public class LocationMap implements ILocationResolver { + private static final IASTName[] EMPTY_NAMES = {}; + + private String fTranslationUnitPath; + private IASTTranslationUnit fTranslationUnit; + + private ArrayList fDirectives= new ArrayList(); + private ArrayList fProblems= new ArrayList(); + private ArrayList fComments= new ArrayList(); + private ArrayList fBuiltinMacros= new ArrayList(); + private IdentityHashMap fMacroExpansions= new IdentityHashMap(); + + private LocationCtx fRootContext= null; + private LocationCtx fCurrentContext= null; + private int fLastChildInsertionOffset; + + // stuff computed on demand + private IdentityHashMap fMacroDefinitionMap= null; + + + + public void registerPredefinedMacro(IMacroBinding macro) { + registerPredefinedMacro(macro, getCurrentFilename(), 0, 0); + } + + public void registerMacroFromIndex(IMacroBinding macro, String filename, int nameOffset, int nameEndOffset, int expansionOffset) { + registerPredefinedMacro(macro, filename, getSequenceNumberForOffset(nameOffset), getSequenceNumberForOffset(nameEndOffset)); + } + + private void registerPredefinedMacro(IMacroBinding macro, String filename, int nameNumber, int nameEndNumber) { + ASTMacro astmacro; + if (macro.isFunctionStyle()) { + astmacro= new ASTFunctionMacro(fTranslationUnit, macro, filename, nameNumber, nameEndNumber); + } + else { + astmacro= new ASTMacro(fTranslationUnit, macro, filename, nameNumber, nameEndNumber); + } + fBuiltinMacros.add(astmacro); + } + + /** + * The outermost context must be a translation unit. You must call this method exactly once and before + * creating any other context. + */ + public ILocationCtx pushTranslationUnit(String filename, char[] buffer) { + assert fCurrentContext == null; + fTranslationUnitPath= filename; + fRootContext= fCurrentContext= new FileLocationCtx(null, filename, buffer, 0, 0, 0, null); + fLastChildInsertionOffset= 0; + return fCurrentContext; + } + + /** + * Starts an artificial context that can be used to include files without having a source that contains + * the include directives. + * @param buffer a buffer containing the include directives. + * @param isMacroFile whether the context is used for running the preprocessor, only. + */ + public ILocationCtx pushPreInclusion(char[] buffer, int offset, boolean isMacroFile) { + assert fCurrentContext != null; + int sequenceNumber= getSequenceNumberForOffset(offset); + fCurrentContext= new ContainerLocationCtx(fCurrentContext, buffer, offset, offset, sequenceNumber); + fLastChildInsertionOffset= 0; + return fCurrentContext; + } + + /** + * Starts a context for an included file. + * @param buffer the buffer containing the content of the inclusion. + * @param filename the filename of the included file + * @param startOffset offset in the current context. + * @param nameOffset offset in the current context. + * @param endOffset offset in the current context + * @param name name of the include without delimiters ("" or <>) + * @param userInclude true when specified with double-quotes. + */ + public ILocationCtx pushInclusion(int startOffset, int nameOffset, int nameEndOffset, int endOffset, + char[] buffer, String filename, char[] name, boolean userInclude) { + assert fCurrentContext != null; + int startNumber= getSequenceNumberForOffset(startOffset); + int nameNumber= getSequenceNumberForOffset(nameOffset); + int nameEndNumber= getSequenceNumberForOffset(nameEndOffset); + int endNumber= getSequenceNumberForOffset(endOffset); + final ASTInclusionStatement inclusionStatement= + new ASTInclusionStatement(fTranslationUnit, startNumber, nameNumber, nameEndNumber, endNumber, name, filename, userInclude, true); + fDirectives.add(inclusionStatement); + fCurrentContext= new FileLocationCtx(fCurrentContext, filename, buffer, startOffset, endOffset, endNumber, inclusionStatement); + fLastChildInsertionOffset= 0; + return fCurrentContext; + } + + /** + * Creates a name representing an implicit macro expansion. The returned name can be fed into + * {@link #pushMacroExpansion(int, int, int, int, IMacroBinding, IASTName[])}. + * @param macro the macro that has been expanded + * @param imageLocationInfo the image-location for the name of the macro. + */ + public IASTName encounterImplicitMacroExpansion(IPreprocessorMacro macro, ImageLocationInfo imageLocationInfo) { + return new ASTMacroReferenceName(fTranslationUnit, macro, imageLocationInfo); + } + + /** + * Creates a new context for the result of a (recursive) macro-expansion. + * @param startOffset offset within the current context where macro-expansion starts. + * @param nameOffset offset within the current context where the name for the macro-expansion starts. + * @param nameEndOffset offset within the current context where the name for the macro-expansion ends. + * @param endOffset offset within the current context where the entire macro-expansion ends. + * @param macro the outermost macro that got expanded. + * @param implicitMacroReferences an array of implicit macro-expansions. + * @param imageLocations an array of image-locations for the new context. + */ + public ILocationCtx pushMacroExpansion(int startOffset, int nameOffset, int nameEndOffset, int endOffset, int contextLength, + IPreprocessorMacro macro, IASTName[] implicitMacroReferences, ImageLocationInfo[] imageLocations) { + int startNumber= getSequenceNumberForOffset(startOffset); + int nameNumber= getSequenceNumberForOffset(nameOffset); + int nameEndNumber= getSequenceNumberForOffset(nameEndOffset); + int endNumber= getSequenceNumberForOffset(endOffset); + + for (int i = 0; i < implicitMacroReferences.length; i++) { + ASTMacroReferenceName name = (ASTMacroReferenceName) implicitMacroReferences[i]; + name.setOffsetAndLength(startNumber, endNumber); + addExpansion((IPreprocessorMacro) name.getBinding(), name); + } + + ASTPreprocessorName expansion= new ASTPreprocessorName(fTranslationUnit, IASTTranslationUnit.EXPANSION_NAME, + nameNumber, nameEndNumber, macro.getNameCharArray(), macro); + addExpansion(macro, expansion); + + fCurrentContext= new MacroExpansionCtx(fCurrentContext, startOffset, endOffset, endNumber, contextLength, imageLocations, expansion); + fLastChildInsertionOffset= 0; + return fCurrentContext; + } + + private void addExpansion(IPreprocessorMacro macro, ASTPreprocessorName name) { + List list= (List) fMacroExpansions.get(macro); + if (list == null) { + list= new ArrayList(); + fMacroExpansions.put(macro, list); + } + list.add(name); + } + + /** + * Ends the current context. + * @param locationCtx the current context, used to check whether caller and location map are still in sync. + */ + public void popContext(ILocationCtx locationCtx) { + assert fCurrentContext == locationCtx; + final LocationCtx child= fCurrentContext; + final LocationCtx parent= fCurrentContext.getParent(); + if (parent != null) { + fCurrentContext= child.getParent(); + fLastChildInsertionOffset= child.fParentEndOffset; + parent.addChildSequenceLength(child.getSequenceLength()); + fCurrentContext= parent; + } + } + + /** + * Reports an inclusion that is not performed. + * @param startOffset offset in the current context. + * @param nameOffset offset in the current context. + * @param endOffset offset in the current context + * @param name name of the include without delimiters ("" or <>) + * @param filename the filename of the included file + * @param userInclude true when specified with double-quotes. + * @param active true when include appears in active code. + */ + public void encounterPoundInclude(int startOffset, int nameOffset, int nameEndOffset, int endOffset, + char[] name, String filename, boolean userInclude, boolean active) { + startOffset= getSequenceNumberForOffset(startOffset); // there may be a macro expansion + nameOffset= getSequenceNumberForOffset(nameOffset); // there may be a macro expansion + nameEndOffset= getSequenceNumberForOffset(nameEndOffset); + endOffset= getSequenceNumberForOffset(endOffset); + fDirectives.add(new ASTInclusionStatement(fTranslationUnit, startOffset, nameOffset, nameEndOffset, endOffset, name, filename, userInclude, active)); + } + + public void encounteredComment(int offset, int endOffset, boolean isBlockComment) { + offset= getSequenceNumberForOffset(offset); + endOffset= getSequenceNumberForOffset(endOffset); + fComments.add(new ASTComment(fTranslationUnit, offset, endOffset, isBlockComment)); + } + + public void encounterProblem(int id, char[] arg, int offset, int endOffset) { + offset= getSequenceNumberForOffset(offset); + endOffset= getSequenceNumberForOffset(endOffset); + ASTProblem problem = new ASTProblem(id, arg, offset, endOffset); + fProblems.add(problem); + } + + public void encounterPoundElse(int startOffset, int endOffset, boolean isActive) { + startOffset= getSequenceNumberForOffset(startOffset); + endOffset= getSequenceNumberForOffset(endOffset); + fDirectives.add(new ASTElse(fTranslationUnit, startOffset, endOffset, isActive)); + } + + public void encounterPoundElif(int startOffset, int condOffset, int condEndOffset, int endOffset, boolean isActive) { + startOffset= getSequenceNumberForOffset(startOffset); // there may be a macro expansion + condOffset= getSequenceNumberForOffset(condOffset); // there may be a macro expansion + condEndOffset= getSequenceNumberForOffset(condEndOffset); + endOffset= getSequenceNumberForOffset(endOffset); + fDirectives.add(new ASTElif(fTranslationUnit, startOffset, condOffset, condEndOffset, endOffset, isActive)); + } + + public void encounterPoundEndIf(int startOffset, int endOffset) { + startOffset= getSequenceNumberForOffset(startOffset); + endOffset= getSequenceNumberForOffset(endOffset); + fDirectives.add(new ASTEndif(fTranslationUnit, startOffset, endOffset)); + } + + public void encounterPoundError(int startOffset, int condOffset, int condEndOffset, int endOffset) { + startOffset= getSequenceNumberForOffset(startOffset); + condOffset= getSequenceNumberForOffset(condOffset); + condEndOffset= getSequenceNumberForOffset(condEndOffset); + endOffset= getSequenceNumberForOffset(endOffset); + fDirectives.add(new ASTError(fTranslationUnit, startOffset, condOffset, condEndOffset, endOffset)); + } + + public void encounterPoundPragma(int startOffset, int condOffset, int condEndOffset, int endOffset) { + startOffset= getSequenceNumberForOffset(startOffset); + condOffset= getSequenceNumberForOffset(condOffset); + condEndOffset= getSequenceNumberForOffset(condEndOffset); + endOffset= getSequenceNumberForOffset(endOffset); + fDirectives.add(new ASTPragma(fTranslationUnit, startOffset, condOffset, condEndOffset, endOffset)); + } + + public void encounterPoundIfdef(int startOffset, int condOffset, int condEndOffset, int endOffset, boolean isActive) { + startOffset= getSequenceNumberForOffset(startOffset); + condOffset= getSequenceNumberForOffset(condOffset); + condEndOffset= getSequenceNumberForOffset(condEndOffset); + endOffset= getSequenceNumberForOffset(endOffset); + fDirectives.add(new ASTIfdef(fTranslationUnit, startOffset, condOffset, condEndOffset, endOffset, isActive)); + } + + public void encounterPoundIfndef(int startOffset, int condOffset, int condEndOffset, int endOffset, boolean isActive) { + startOffset= getSequenceNumberForOffset(startOffset); + condOffset= getSequenceNumberForOffset(condOffset); + condEndOffset= getSequenceNumberForOffset(condEndOffset); + endOffset= getSequenceNumberForOffset(endOffset); + fDirectives.add(new ASTIfndef(fTranslationUnit, startOffset, condOffset, condEndOffset, endOffset, isActive)); + } + + public void encounterPoundIf(int startOffset, int condOffset, int condEndOffset, int endOffset, boolean isActive) { + startOffset= getSequenceNumberForOffset(startOffset); // there may be a macro expansion + condOffset= getSequenceNumberForOffset(condOffset); // there may be a macro expansion + condEndOffset= getSequenceNumberForOffset(condEndOffset); + endOffset= getSequenceNumberForOffset(endOffset); + fDirectives.add(new ASTIf(fTranslationUnit, startOffset, condOffset, condEndOffset, endOffset, isActive)); + } + + public void encounterPoundDefine(int startOffset, int nameOffset, int nameEndOffset, int expansionOffset, int endOffset, IMacroBinding macrodef) { + startOffset= getSequenceNumberForOffset(startOffset); + nameOffset= getSequenceNumberForOffset(nameOffset); + nameEndOffset= getSequenceNumberForOffset(nameEndOffset); + expansionOffset= getSequenceNumberForOffset(expansionOffset); + endOffset= getSequenceNumberForOffset(endOffset); + ASTPreprocessorNode astMacro; + if (!macrodef.isFunctionStyle()) { + astMacro= new ASTMacro(fTranslationUnit, macrodef, startOffset, nameOffset, nameEndOffset, expansionOffset, endOffset); + } + else { + astMacro= new ASTFunctionMacro(fTranslationUnit, macrodef, startOffset, nameOffset, nameEndOffset, expansionOffset, endOffset); + } + fDirectives.add(astMacro); + } + + public void encounterPoundUndef(PreprocessorMacro definition, int startOffset, int nameOffset, int nameEndOffset, int endOffset, char[] name) { + startOffset= getSequenceNumberForOffset(startOffset); + nameOffset= getSequenceNumberForOffset(nameOffset); + nameEndOffset= getSequenceNumberForOffset(nameEndOffset); + endOffset= getSequenceNumberForOffset(endOffset); + fDirectives.add(new ASTUndef(fTranslationUnit, name, startOffset, nameOffset, nameEndOffset, endOffset)); + } + + public void setRootNode(IASTTranslationUnit root) { + fTranslationUnit= root; + } + + public String getTranslationUnitPath() { + return fTranslationUnitPath; + } + + /** + * Line number of offset in current context. + * @param offset in current context. + */ + public int getCurrentLineNumber(int offset) { + return fCurrentContext.getLineNumber(offset); + } + + /** + * Returns the filename of the current context. If the context is a macro-expansion the filename of + * the enclosing file is returned. + */ + public String getCurrentFilename() { + return fCurrentContext.getFilename(); + } + + /** + * Returns the sequence number corresponding to the offset in the current context. + *

+ * You must insert all child contexts before the given offset before conversion. + */ + private int getSequenceNumberForOffset(int offset) { + return fCurrentContext.getSequenceNumberForOffset(offset, offset < fLastChildInsertionOffset); + } + + public String getContainingFilename(int sequenceNumber) { + LocationCtx ctx= fRootContext.findContextForSequenceNumberRange(sequenceNumber, 1); + return new String(ctx.getFilename()); + } + + public IASTFileLocation getMappedFileLocation(int sequenceNumber, int length) { + return fRootContext.getFileLocationForSequenceNumberRange(sequenceNumber, length); + } + + public IASTPreprocessorMacroDefinition[] getMacroDefinitions() { + ArrayList result= new ArrayList(); + for (Iterator iterator = fDirectives.iterator(); iterator.hasNext();) { + Object directive= iterator.next(); + if (directive instanceof IASTPreprocessorMacroDefinition) { + result.add(directive); + } + } + return (IASTPreprocessorMacroDefinition[]) result.toArray(new IASTPreprocessorMacroDefinition[result.size()]); + } + + public IASTPreprocessorIncludeStatement[] getIncludeDirectives() { + ArrayList result= new ArrayList(); + for (Iterator iterator = fDirectives.iterator(); iterator.hasNext();) { + Object directive= iterator.next(); + if (directive instanceof IASTPreprocessorIncludeStatement) { + result.add(directive); + } + } + return (IASTPreprocessorIncludeStatement[]) result.toArray(new IASTPreprocessorIncludeStatement[result.size()]); + } + + public IASTPreprocessorStatement[] getAllPreprocessorStatements() { + return (IASTPreprocessorStatement[]) fDirectives.toArray(new IASTPreprocessorStatement[fDirectives.size()]); + } + + public IASTPreprocessorMacroDefinition[] getBuiltinMacroDefinitions() { + return (IASTPreprocessorMacroDefinition[]) fBuiltinMacros.toArray(new IASTPreprocessorMacroDefinition[fBuiltinMacros.size()]); + } + + public IASTProblem[] getScannerProblems() { + return (IASTProblem[]) fProblems.toArray(new IASTProblem[fProblems.size()]); + } + + + public IASTName[] getDeclarations(IMacroBinding binding) { + if (fMacroDefinitionMap == null) { + fMacroDefinitionMap= new IdentityHashMap(); + IASTPreprocessorMacroDefinition[] defs= getMacroDefinitions(); + for (int i = 0; i < defs.length; i++) { + final IASTName name = defs[i].getName(); + if (name != null) { + fMacroDefinitionMap.put(name.getBinding(), name); + } + } + } + IASTName name= (IASTName) fMacroDefinitionMap.get(binding); + return name == null ? EMPTY_NAMES : new IASTName[] {name}; + } + + public IASTName[] getReferences(IMacroBinding binding) { + List list= (List) fMacroExpansions.get(binding); + if (list == null) { + return EMPTY_NAMES; + } + return (IASTName[]) list.toArray(new IASTName[list.size()]); + } + + public IDependencyTree getDependencyTree() { + return new DependencyTree(fRootContext); + } + + // stuff to remove from ILocationResolver + public IASTName[] getMacroExpansions() { + throw new UnsupportedOperationException(); + } + public void cleanup() { + throw new UnsupportedOperationException(); + } + public IASTFileLocation flattenLocations(IASTNodeLocation[] nodeLocations) { + throw new UnsupportedOperationException(); + } + public IASTNodeLocation[] getLocations(int offset, int length) { + throw new UnsupportedOperationException(); + } + public ASTPreprocessorSelectionResult getPreprocessorNode(String path, int offset, int length) { + throw new UnsupportedOperationException(); + } + public char[] getUnpreprocessedSignature(IASTNodeLocation[] locations) { + throw new UnsupportedOperationException(); + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroDefinitionParser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroDefinitionParser.java new file mode 100644 index 00000000000..d874b21033b --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroDefinitionParser.java @@ -0,0 +1,249 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import java.util.ArrayList; + +import org.eclipse.cdt.core.parser.IProblem; +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.Keywords; +import org.eclipse.cdt.core.parser.OffsetLimitReachedException; +import org.eclipse.cdt.core.parser.util.CharArrayUtils; + +/** + * Utility to parse macro definitions and create the macro objects for the preprocessor. + * @since 5.0 + */ +class MacroDefinitionParser { + static class InvalidMacroDefinitionException extends Exception { + public char[] fName; + public InvalidMacroDefinitionException(char[] name) { + fName= name; + } + } + + private static final int ORIGIN_PREPROCESSOR_DIRECTIVE = OffsetLimitReachedException.ORIGIN_PREPROCESSOR_DIRECTIVE; + + private int fHasVarArgs; + private int fExpansionOffset; + private int fExpansionEndOffset; + private Token fNameToken; + + /** + * In case the name was successfully parsed, the name token is returned. + * Otherwise the return value is undefined. + */ + public Token getNameToken() { + return fNameToken; + } + + /** + * In case the expansion was successfully parsed, the start offset is returned. + * Otherwise the return value is undefined. + */ + public int getExpansionOffset() { + return fExpansionOffset; + } + + /** + * In case the expansion was successfully parsed, the end offset is returned. + * Otherwise the return value is undefined. + */ + public int getExpansionEndOffset() { + return fExpansionEndOffset; + } + + /** + * Parses an entire macro definition. Name must be the next token of the lexer. + */ + public ObjectStyleMacro parseMacroDefinition(final Lexer lexer, final ILexerLog log) + throws OffsetLimitReachedException, InvalidMacroDefinitionException { + final Token name = parseName(lexer); + final char[] source= lexer.getInput(); + final char[] nameChars= name.getCharImage(); + final char[][] paramList= parseParamList(lexer, name); + final Token replacement= parseExpansion(lexer, log, nameChars, paramList, fHasVarArgs); + if (paramList == null) { + return new ObjectStyleMacro(nameChars, fExpansionOffset, fExpansionEndOffset, replacement, source); + } + return new FunctionStyleMacro(nameChars, paramList, fHasVarArgs, fExpansionOffset, fExpansionEndOffset, replacement, source); + } + + /** + * Parses a macro definition without the replacement. Name must be the next token of the lexer. + */ + public PreprocessorMacro parseMacroDefinition(final Lexer lexer, final ILexerLog log, final char[] replacement) + throws InvalidMacroDefinitionException, OffsetLimitReachedException { + final Token name = parseName(lexer); + + final char[] nameChars = name.getCharImage(); + final char[][] paramList= parseParamList(lexer, name); + final Token replacementToken = lexer.currentToken(); + if (replacementToken.getType() != Lexer.tEND_OF_INPUT) { + throw new InvalidMacroDefinitionException(nameChars); + } + + if (paramList == null) { + return new ObjectStyleMacro(nameChars, replacement); + } + return new FunctionStyleMacro(nameChars, paramList, fHasVarArgs, replacement); + } + + /** + * Parses a macro definition basically checking for var-args. + */ + public PreprocessorMacro parseMacroDefinition(final char[] name, char[][] paramList, final char[] replacement) { + final int length = paramList.length; + fHasVarArgs= 0; + if (paramList != null && length > 0) { + char[] lastParam= paramList[length-1]; + final int lpl = lastParam.length; + switch(lpl) { + case 0: case 1: case 2: + break; + case 3: + if (CharArrayUtils.equals(lastParam, Keywords.cpELLIPSIS)) { + fHasVarArgs= FunctionStyleMacro.VAARGS; + char[][] copy= new char[length][]; + System.arraycopy(paramList, 0, copy, 0, length-1); + copy[length-1]= Keywords.cVA_ARGS; + paramList= copy; + } + break; + default: + if (CharArrayUtils.equals(lastParam, lpl-3, 3, Keywords.cpELLIPSIS)) { + fHasVarArgs= FunctionStyleMacro.NAMED_VAARGS; + char[][] copy= new char[length][]; + System.arraycopy(paramList, 0, copy, 0, length-1); + copy[length-1]= CharArrayUtils.subarray(lastParam, 0, lpl-3); + paramList= copy; + } + break; + } + } + + if (paramList == null) { + return new ObjectStyleMacro(name, replacement); + } + return new FunctionStyleMacro(name, paramList, fHasVarArgs, replacement); + } + + private Token parseName(final Lexer lexer) throws OffsetLimitReachedException, InvalidMacroDefinitionException { + final Token name= lexer.nextToken(); + final int tt= name.getType(); + if (tt != IToken.tIDENTIFIER) { + if (tt == IToken.tCOMPLETION) { + throw new OffsetLimitReachedException(ORIGIN_PREPROCESSOR_DIRECTIVE, name); + } + throw new InvalidMacroDefinitionException(name.getCharImage()); + } + fNameToken= name; + return name; + } + + private char[][] parseParamList(Lexer lex, final Token name) throws OffsetLimitReachedException, InvalidMacroDefinitionException { + final Token lparen= lex.nextToken(); + fHasVarArgs= FunctionStyleMacro.NO_VAARGS; + if (lparen.getType() != IToken.tLPAREN || name.getEndOffset() != lparen.getOffset()) { + return null; + } + ArrayList paramList= new ArrayList(); + IToken next= null; + do { + final Token param= lex.nextToken(); + switch (param.getType()) { + case IToken.tCOMPLETION: + throw new OffsetLimitReachedException(ORIGIN_PREPROCESSOR_DIRECTIVE, param); + + case IToken.tIDENTIFIER: + paramList.add(param.getCharImage()); + next= lex.nextToken(); + if (next.getType() == IToken.tELLIPSIS) { + fHasVarArgs= FunctionStyleMacro.NAMED_VAARGS; + next= lex.nextToken(); + } + break; + + case IToken.tELLIPSIS: + fHasVarArgs= FunctionStyleMacro.VAARGS; + paramList.add(Keywords.cVA_ARGS); + next= lex.nextToken(); + break; + + default: + throw new InvalidMacroDefinitionException(name.getCharImage()); + } + } + while (fHasVarArgs==0 && next.getType() == IToken.tCOMMA); + if (next.getType() != IToken.tRPAREN) { + throw new InvalidMacroDefinitionException(name.getCharImage()); + } + next= lex.nextToken(); // consume the closing parenthesis + + return (char[][]) paramList.toArray(new char[paramList.size()][]); + } + + private Token parseExpansion(final Lexer lexer, final ILexerLog log, final char[] name, final char[][] paramList, final int hasVarArgs) + throws OffsetLimitReachedException { + final boolean allowVarArgsArray= hasVarArgs==FunctionStyleMacro.VAARGS; + boolean needParam= false; + Token needAnotherToken= null; + + Token candidate= lexer.currentToken(); + fExpansionOffset= candidate.getOffset(); + Token last= new SimpleToken(Lexer.tNEWLINE, fExpansionOffset, fExpansionOffset); + final Token resultHolder= last; + + loop: while(true) { + switch(candidate.getType()) { + case IToken.tCOMPLETION: + throw new OffsetLimitReachedException(ORIGIN_PREPROCESSOR_DIRECTIVE, candidate); + case Lexer.tEND_OF_INPUT: + case Lexer.tNEWLINE: + break loop; + case IToken.tIDENTIFIER: + if (!allowVarArgsArray && CharArrayUtils.equals(Keywords.cVA_ARGS, candidate.getCharImage())) { + log.handleProblem(IProblem.PREPROCESSOR_INVALID_VA_ARGS, null, fExpansionOffset, candidate.getEndOffset()); + } + if (needParam && CharArrayUtils.indexOf(candidate.getCharImage(), paramList) == -1) { + log.handleProblem(IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, name, fExpansionOffset, candidate.getEndOffset()); + } + needParam= false; + needAnotherToken= null; + break; + case IToken.tPOUND: + needParam= paramList != null; + break; + case IToken.tPOUNDPOUND: + if (needParam || resultHolder == last) { + log.handleProblem(IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, name, fExpansionOffset, candidate.getEndOffset()); + } + needAnotherToken= candidate; + needParam= false; + break; + default: + if (needParam) { + log.handleProblem(IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, name, fExpansionOffset, candidate.getEndOffset()); + needParam= false; + } + needAnotherToken= null; + break; + } + last.setNext(candidate); last=candidate; + candidate= lexer.nextToken(); + } + if (needAnotherToken != null) { + log.handleProblem(IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, name, needAnotherToken.getOffset(), needAnotherToken.getEndOffset()); + } + fExpansionEndOffset= last.getEndOffset(); + return (Token) resultHolder.getNext(); + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/PreprocessorMacro.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/PreprocessorMacro.java new file mode 100644 index 00000000000..f9eb6a55e3c --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/PreprocessorMacro.java @@ -0,0 +1,183 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import org.eclipse.cdt.core.dom.ILinkage; +import org.eclipse.cdt.core.dom.ast.IScope; +import org.eclipse.cdt.core.parser.Keywords; +import org.eclipse.cdt.core.parser.util.CharArrayUtils; +import org.eclipse.cdt.internal.core.dom.Linkage; + +/** + * Models macros used by the preprocessor + * @since 5.0 + */ +abstract class PreprocessorMacro implements IPreprocessorMacro { + final private char[] fName; + + public PreprocessorMacro(char[] name) { + fName= name; + } + + final public ILinkage getLinkage() { + return Linkage.NO_LINKAGE; + } + + final public char[] getNameCharArray() { + return fName; + } + + final public String getName() { + return new String(fName); + } + + public IScope getScope() { + return null; + } + + public boolean isFunctionStyle() { + return false; + } + + public Object getAdapter(Class clazz) { + return null; + } +} + +abstract class DynamicStyleMacro extends PreprocessorMacro { + + public DynamicStyleMacro(char[] name) { + super(name); + } + public char[] getExpansion() { + return execute().getCharImage(); + } + public abstract Token execute(); +} + +class ObjectStyleMacro extends PreprocessorMacro { + private static final Token NOT_INITIALIZED = new SimpleToken(0,0,0); + + private final char[] fExpansion; + final int fExpansionOffset; + final int fEndOffset; +// private Token fExpansionTokens; + + public ObjectStyleMacro(char[] name, char[] expansion) { + this(name, 0, expansion.length, NOT_INITIALIZED, expansion); + } + + public ObjectStyleMacro(char[] name, int expansionOffset, int endOffset, Token expansion, char[] source) { + super(name); + fExpansionOffset= expansionOffset; + fEndOffset= endOffset; + fExpansion= source; +// fExpansionTokens= expansion; + } + + public int findParameter(char[] tokenImage) { + return -1; + } + + public char[] getExpansion() { + final int length = fEndOffset - fExpansionOffset; + if (length == fExpansion.length) { + return fExpansion; + } + char[] result= new char[length]; + System.arraycopy(fExpansion, fEndOffset, result, 0, length); + return result; + } +} + + +class FunctionStyleMacro extends ObjectStyleMacro { + public static final int NO_VAARGS = 0; // M(a) + public static final int VAARGS = 1; // M(...) + public static final int NAMED_VAARGS= 2; // M(a...) + + final private char[][] fParamList; + final private int fHasVarArgs; + private char[] fSignature; + + public FunctionStyleMacro(char[] name, char[][] paramList, int hasVarArgs, char[] expansion) { + super(name, expansion); + fParamList = paramList; + fHasVarArgs= hasVarArgs; + } + + public FunctionStyleMacro(char[] name, char[][] paramList, int hasVarArgs, int expansionFileOffset, int endFileOffset, + Token expansion, char[] source) { + super(name, expansionFileOffset, endFileOffset, expansion, source); + fParamList = paramList; + fHasVarArgs= hasVarArgs; + } + + public char[][] getParamList() { + return fParamList; + } + + public char[] getSignature() { + if (fSignature != null) { + return fSignature; + } + + StringBuffer result= new StringBuffer(); + result.append(getName()); + result.append('('); + + final int lastIdx= fParamList.length-1; + if (lastIdx >= 0) { + for (int i = 0; i < lastIdx; i++) { + result.append(fParamList[i]); + result.append(','); + } + switch(fHasVarArgs) { + case VAARGS: + result.append(Keywords.cpELLIPSIS); + break; + case NAMED_VAARGS: + result.append(fParamList[lastIdx]); + result.append(Keywords.cpELLIPSIS); + break; + default: + result.append(fParamList[lastIdx]); + break; + } + } + result.append(')'); + final int len= result.length(); + final char[] sig= new char[len]; + result.getChars(0, len, sig, 0); + fSignature= sig; + return sig; + } + + /** + * Returns one of {@link #NO_VAARGS}, {@link #VAARGS} or {@link #NAMED_VAARGS}. + */ + public int hasVarArgs() { + return fHasVarArgs; + } + + public int findParameter(final char[] identifier) { + for (int i=0; i < fParamList.length; i++) { + if (CharArrayUtils.equals(fParamList[i], identifier)) { + return i; + } + } + return -1; + } + + public boolean isFunctionStyle() { + return true; + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContext.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContext.java new file mode 100644 index 00000000000..49e1fed4b5d --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContext.java @@ -0,0 +1,79 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import org.eclipse.cdt.core.parser.OffsetLimitReachedException; + + + +/** + * Represents part of the input to the preprocessor. This may be a file or the result of a macro expansion. + * @since 5.0 + */ +abstract class ScannerContext { + public static final Integer BRANCH_IF = new Integer(0); + public static final Integer BRANCH_ELIF = new Integer(1); + public static final Integer BRANCH_ELSE = new Integer(2); + public static final Integer BRANCH_END = new Integer(3); + + private final ILocationCtx fLocationCtx; + private final ScannerContext fParent; + + /** + * @param ctx + * @param parent context to be used after this context is done. + */ + public ScannerContext(ILocationCtx ctx, ScannerContext parent) { + fLocationCtx= ctx; + fParent= parent; + } + + /** + * Returns the location context associated with this scanner context. + */ + public final ILocationCtx getLocationCtx() { + return fLocationCtx; + } + + /** + * Returns the parent context which will be used after this context is finished. + * May return null. + */ + public final ScannerContext getParent() { + return fParent; + } + + /** + * Tests whether or not the current identifier of this context are subject to macro-expansion. + */ + public boolean expandsMacros() { + return true; + } + + /** + * Returns the lexer for a preprocessing directive or null if the current + * token is not the start of a preprocessing directive. + *

+ * The current token starts a directive, whenever the context supports directives, + * and the current token is a pound that occurs as the first token on the line. + */ + public abstract Lexer getLexerForPPDirective(); + + /** + * Needs to be called whenever we change over to another branch of conditional + * compilation. Returns whether the change is legal at this point or not. + */ + public abstract boolean changeBranch(Integer state); + + + public abstract Token currentPPToken(); + public abstract Token nextPPToken() throws OffsetLimitReachedException; +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextFile.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextFile.java new file mode 100644 index 00000000000..33643e4d4d5 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextFile.java @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import java.util.ArrayList; + +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.OffsetLimitReachedException; + +/** + * Wraps a {@link Lexer} and provides additional information for the preprocessor. + *

+ * Note that for parsing the preprocessor directives the lexer is used directly, so this class + * is not allowed to store any state about the lexing process. + * + * since 5.0 + */ +public class ScannerContextFile extends ScannerContext { + + private final Lexer fLexer; + private final ArrayList fBranches= new ArrayList(); + + public ScannerContextFile(ILocationCtx ctx, ScannerContext parent, Lexer lexer) { + super(ctx, parent); + fLexer= lexer; + } + + public Token currentPPToken() { + return fLexer.currentToken(); + } + + public Token nextPPToken() throws OffsetLimitReachedException { + return fLexer.nextToken(); + } + + public Lexer getLexerForPPDirective() { + if (fLexer.currentTokenIsFirstOnLine() && fLexer.currentToken().getType() == IToken.tPOUND) { + return fLexer; + } + return null; + } + + public boolean changeBranch(Integer branchKind) { + // an if starts a new conditional construct + if (branchKind == BRANCH_IF) { + fBranches.add(branchKind); + return true; + } + // if we are not inside of an conditional there shouldn't be an #else, #elsif or #end + final int pos= fBranches.size()-1; + if (pos < 0) { + return false; + } + // an #end just pops one construct. + if (branchKind == BRANCH_END) { + fBranches.remove(pos); + return true; + } + // #elsif or #else cannot appear after another #else + if (fBranches.get(pos) == BRANCH_ELSE) { + return false; + } + // overwrite #if, #elsif with #elsif or #else + fBranches.set(pos, branchKind); + return true; + } +} \ No newline at end of file diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacroFile.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacroFile.java new file mode 100644 index 00000000000..cf0f8d85343 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacroFile.java @@ -0,0 +1,46 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner; + +import org.eclipse.cdt.core.parser.OffsetLimitReachedException; + +/** + * Context used to run the preprocessor while swallowing all tokens. + * Needed to process macro-files as specified by the -imacro compiler option of gcc. + * @since 5.0 + */ +public class ScannerContextMacroFile extends ScannerContextFile { + private final CPreprocessor fCpp; + private boolean fSkippingTokens= false; + + public ScannerContextMacroFile(CPreprocessor cpp, ILocationCtx ctx, ScannerContext parent, Lexer lexer) { + super(ctx, parent, lexer); + fCpp= cpp; + } + + public Token nextPPToken() throws OffsetLimitReachedException { + if (fSkippingTokens) { + final Token t= super.nextPPToken(); + if (t.getType() == Lexer.tEND_OF_INPUT) { + fSkippingTokens= false; + } + return t; + } + + // use preprocessor to read tokens off this context, until this context is done. + fSkippingTokens= true; + Token t; + do { + t= fCpp.fetchTokenFromPreprocessor(); + } while (fSkippingTokens); + return t; + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextPPDirective.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextPPDirective.java new file mode 100644 index 00000000000..e4d02087686 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextPPDirective.java @@ -0,0 +1,119 @@ +/******************************************************************************* + * Copyright (c) 2007 Wind River Systems, Inc. and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Markus Schorn - initial API and implementation + *******************************************************************************/ + +package org.eclipse.cdt.internal.core.parser.scanner; + +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.Keywords; +import org.eclipse.cdt.core.parser.OffsetLimitReachedException; +import org.eclipse.cdt.core.parser.util.CharArrayUtils; + +/** + * Wraps a ScannerContext and modifies its behavior by limiting the tokens + * to the ones on the current line. Instead of the newline token an end-of-input + * token is returned. The newline token of the underlying context is not consumed. + * @since 5.0 + */ +public final class ScannerContextPPDirective extends ScannerContext { + + private static final int STATE_PREVENT_EXPANSION = 1; + private static final int STATE_DEFINED_LPAREN = 2; + private static final int STATE_DEFINED = 3; + private final Lexer fLexer; + private Token fToken; + private boolean fConvertDefinedToken; + private int fPreventMacroExpansion= 0; + private int fLastEndOffset; + + public ScannerContextPPDirective(Lexer lexer, boolean convertDefinedToken) { + super(null, null); + fLexer= lexer; + final Token currentToken = lexer.currentToken(); + fLastEndOffset= currentToken.getOffset(); + fToken= convertToken(currentToken); + fConvertDefinedToken= convertDefinedToken; + } + + public Token currentPPToken() { + return fToken; + } + + public Token nextPPToken() throws OffsetLimitReachedException { + if (fToken.getType() == Lexer.tEND_OF_INPUT) { + return fToken; + } + Token t1= fLexer.nextToken(); + t1 = convertToken(t1); + fToken= t1; + + Token t = t1; + return t; + } + + public Lexer getLexerForPPDirective() { + return null; + } + + public boolean changeBranch(Integer state) { + return false; + } + + private Token convertToken(Token t) { + switch (t.getType()) { + case Lexer.tNEWLINE: + t= new SimpleToken(Lexer.tEND_OF_INPUT, fToken.getEndOffset(), fToken.getEndOffset()); + break; + case IToken.tIDENTIFIER: + if (fConvertDefinedToken && CharArrayUtils.equals(Keywords.cDEFINED, fToken.getCharImage())) { + t.setType(CPreprocessor.tDEFINED); + fPreventMacroExpansion= STATE_DEFINED; + } + else { + switch(fPreventMacroExpansion) { + case STATE_DEFINED: + case STATE_DEFINED_LPAREN: + fPreventMacroExpansion= STATE_PREVENT_EXPANSION; + break; + default: + fPreventMacroExpansion= 0; + } + } + fLastEndOffset= t.getEndOffset(); + break; + case IToken.tLPAREN: + if (fPreventMacroExpansion == STATE_DEFINED) { + fPreventMacroExpansion= STATE_DEFINED_LPAREN; // suppress macro-expansion for 'defined (id)' + } + else { + fPreventMacroExpansion= 0; + } + fLastEndOffset= t.getEndOffset(); + break; + default: + fPreventMacroExpansion= 0; + fLastEndOffset= t.getEndOffset(); + break; + } + return t; + } + + public boolean expandsMacros() { + return fPreventMacroExpansion == 0; + } + + public void setInsideIncludeDirective() { + fLexer.setInsideIncludeDirective(); + } + + public int getLastEndOffset() { + return fLastEndOffset; + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Token.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Token.java index 0d008e3cf58..9f68d6d5c41 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Token.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Token.java @@ -12,14 +12,17 @@ package org.eclipse.cdt.internal.core.parser.scanner; import org.eclipse.cdt.core.parser.IToken; - +/** + * Represents tokens found by the lexer. The preprocessor reuses the tokens and passes + * them on to the parsers. + * @since 5.0 + */ public abstract class Token implements IToken { private int fKind; - - int fOffset; - int fEndOffset; + private int fOffset; + private int fEndOffset; - private IToken fNextGrammarToken; + private IToken fNextToken; Token(int kind, int offset, int endOffset) { fKind= kind; @@ -43,46 +46,31 @@ public abstract class Token implements IToken { return fEndOffset-fOffset; } - public IToken getNext() { - return fNextGrammarToken; + return fNextToken; } - - public abstract char[] getTokenImage(); - // for the preprocessor to classify preprocessor tokens public void setType(int kind) { - // mstodo make non-public fKind= kind; } - - // for the preprocessor to chain the tokens + public void setNext(IToken t) { - // mstodo make non-public - fNextGrammarToken= t; + fNextToken= t; } + public abstract char[] getCharImage(); - - + public boolean isOperator() { - // mstodo return TokenUtil.isOperator(fKind); } - public char[] getCharImage() { - // mstodo - throw new UnsupportedOperationException(); - } - public String getImage() { - // mstodo - throw new UnsupportedOperationException(); + return new String(getCharImage()); } - public char[] getFilename() { // mstodo throw new UnsupportedOperationException(); @@ -107,5 +95,59 @@ public abstract class Token implements IToken { // mstodo throw new UnsupportedOperationException(); } - } + +class SimpleToken extends Token { + public SimpleToken(int kind, int offset, int endOffset) { + super(kind, offset, endOffset); + } + + public char[] getCharImage() { + return TokenUtil.getImage(getType()); + } +} + +class DigraphToken extends Token { + public DigraphToken(int kind, int offset, int endOffset) { + super(kind, offset, endOffset); + } + + public char[] getCharImage() { + return TokenUtil.getDigraphImage(getType()); + } +} + +class ImageToken extends Token { + private char[] fImage; + + public ImageToken(int kind, int offset, int endOffset, char[] image) { + super(kind, offset, endOffset); + fImage= image; + } + + public char[] getCharImage() { + return fImage; + } +} + +class SourceImageToken extends Token { + + private char[] fSource; + private char[] fImage; + + public SourceImageToken(int kind, int offset, int endOffset, char[] source) { + super(kind, offset, endOffset); + fSource= source; + } + + public char[] getCharImage() { + if (fImage == null) { + final int length= getLength(); + fImage= new char[length]; + System.arraycopy(fSource, getOffset(), fImage, 0, length); + } + return fImage; + } +} + + diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenUtil.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenUtil.java index 31f4b4248f8..a0412ec98cd 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenUtil.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenUtil.java @@ -124,7 +124,6 @@ public class TokenUtil { case IGCCToken.tMAX: return Keywords.cpMAX; default: - assert false: type; return IMAGE_EMPTY; } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenWithImage.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenWithImage.java deleted file mode 100644 index 0f75aea75ed..00000000000 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/TokenWithImage.java +++ /dev/null @@ -1,38 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2007 Wind River Systems, Inc. and others. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * Markus Schorn - initial API and implementation - *******************************************************************************/ -package org.eclipse.cdt.internal.core.parser.scanner; - -class TokenWithImage extends Token { - - final private Lexer fLexer; - final private int fImageLength; - private char[] fImage; - - public TokenWithImage(int kind, Lexer source, int offset, int endOffset, int imageLength) { - super(kind, offset, endOffset); - fLexer= source; - fImageLength= imageLength; - } - - public TokenWithImage(int kind, int offset, int endOffset, char[] image) { - super(kind, offset, endOffset); - fLexer= null; - fImageLength= 0; - fImage= image; - } - - public char[] getTokenImage() { - if (fImage == null) { - fImage= fLexer.getTokenImage(fOffset, fEndOffset, fImageLength); - } - return fImage; - } -}