Bug 566918 - [C++17] Support for __has_include (standard) and

__has_include_next (extension)

__has_include evaluates whether of the header name passed as parameter
exists. This can only be evaluated as part of a #if directive.
Interestingly, it also has to be reported as defined, i.e. #if
defined(__has_include) or #ifdef. In order to report this as defined,
this implementation adds it as a macro but during macro expansion, it's
actually converted as a dedicated token type. Then this token gets
evaluated during normal preprocessor expression evaluation.

In order to parse header names, there were several options. The main
problem is that header tokens (tQUOTE_HEADER_NAME, tSYSTEM_HEADER_NAME)
are actually produced by the Lexer as part of a special mode
(setInsideIncludeDirective) set during the handling of #include. For
expression evaluation, the tokens are already generated without
setInsideIncludeDirective therefore we only have plain string
and < > tokens.

One approach would be to generate header tokens "earlier" than executing
we need to track a new state while fetching token to configure the Lexer
(setInsideIncludeDirective) when in the context of an __has_include.
There are also complications due to macro expansion within the
__has_include where after one expansion, we don't have a lexer in the
context anymore, introducing more changes.

Another approach would be to remove the Header token creation from the
Lexer itself and let the preprocessor assemble the tokens into an header
string, in both cases of #include and __has_include. This mostly works
and is the approach used in Clang, but the problem is that whereas Clang
keeps track of leading spaces of tokens, CDT doesn't. This means with
such change that CDT would now allow #include <  iostream  > (notice the
white space). I think this is too big of a downside and also too big of
a change to introduce this handling of whitespace at the token level.

The approach used here is more conservative and isolated but also shares
less common logic with #include processing. The non-header token
(string, <, etc) are assembled into a header string only in the case of
a __has_include. So a downside will be that #include and __has_include
will be inconsistent in regards of leading/trailing space parsing but I
feel like this is better than making #include more permissive.

Change-Id: I5b9f5c616c8d999e0c916a85b41f96e20037b651
Signed-off-by: Marc-Andre Laperle <malaperle@gmail.com>

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/InclusionTests.java

@@ -16,10 +16,13 @@
 package org.eclipse.cdt.core.parser.tests.scanner;
 
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.eclipse.cdt.core.model.ICProject;
 import org.eclipse.cdt.core.parser.ExtendedScannerInfo;
 import org.eclipse.cdt.core.parser.FileContent;
+import org.eclipse.cdt.core.parser.IProblem;
 import org.eclipse.cdt.core.parser.IScannerInfo;
 import org.eclipse.cdt.core.parser.IToken;
 import org.eclipse.cdt.core.parser.ParserLanguage;
@@ -261,6 +264,136 @@ public class InclusionTests extends PreprocessorTestsBase {
 		validateEOF();
 	}
 
+	// #if __has_include("does_not_exist.h")
+	// inactive
+	// #endif
+	// #if !__has_include("does_not_exist.h")
+	// identifier
+	// #endif
+	//
+	// #if __has_include("test.h")
+	// identifier2
+	// #endif
+	// #if !__has_include("test.h")
+	// inactive
+	// #endif
+	//
+	// #if __has_include(<test.h>)
+	// identifier3
+	// #endif
+	//
+	// #define MACRO __has_include("test.h")
+	// #if MACRO
+	// identifier4
+	// #endif
+	//
+	// #define MACRO2 __has_include(<test.h>)
+	// #if MACRO2
+	// identifier5
+	// #endif
+	//
+	// #define HEADER_NAME "test.h"
+	// #define MACRO3 __has_include(HEADER_NAME)
+	// #if MACRO3
+	// identifier6
+	// #endif
+	//
+	// //Note: This one works with Clang and MSVC but not GCC.
+	// #define HEADER_NAME2 ("test.h")
+	// #define MACRO4 __has_include HEADER_NAME2
+	// #if MACRO4
+	// identifier7
+	// #endif
+	//
+	// #ifdef __has_include
+	// identifier8
+	// #endif
+	//
+	// #if defined(__has_include)
+	// identifier9
+	// #endif
+	public void testHasInclude() throws Exception {
+		importFile("test.h", "");
+		IFile base = importFile("test.cpp", getAboveComment());
+		IScannerInfo scannerInfo = new ExtendedScannerInfo(Collections.EMPTY_MAP,
+				new String[] { fProject.getProject().getLocation().toOSString() }, new String[] {}, null);
+		FileContent reader = FileContent.create(base);
+		initializeScanner(reader, ParserLanguage.CPP, ParserMode.COMPLETE_PARSE, scannerInfo);
+		validateIdentifier("identifier");
+		validateIdentifier("identifier2");
+		validateIdentifier("identifier3");
+		validateIdentifier("identifier4");
+		validateIdentifier("identifier5");
+		validateIdentifier("identifier6");
+		validateIdentifier("identifier7");
+		validateIdentifier("identifier8");
+		validateIdentifier("identifier9");
+		validateEOF();
+	}
+
+	// #include "foo.h"
+	//
+	// #ifdef __has_include_next
+	// identifier4
+	// #endif
+	//
+	// #if defined(__has_include_next)
+	// identifier5
+	// #endif
+
+	// identifier
+	// #include "intermed.h"
+
+	// identifier2
+	// #if __has_include_next(<foo.h>)
+	// #include_next <foo.h>
+	// #endif
+
+	// identifier3
+	public void testHasIncludeNext() throws Exception {
+		StringBuilder[] sections = getTestContent(4);
+		String baseFile = sections[0].toString(); //$NON-NLS-1$
+		String foo1 = sections[1].toString(); //$NON-NLS-1$
+		String intermed = sections[2].toString(); //$NON-NLS-1$
+		String foo2 = sections[3].toString(); //$NON-NLS-1$
+
+		IFolder one = importFolder("one"); //$NON-NLS-1$
+		IFolder two = importFolder("two"); //$NON-NLS-1$
+		IFile base = importFile("base.cpp", baseFile); //$NON-NLS-1$
+		importFile("one/foo.h", foo1); //$NON-NLS-1$
+		importFile("one/intermed.h", intermed); //$NON-NLS-1$
+		importFile("two/foo.h", foo2); //$NON-NLS-1$
+
+		String[] path = new String[2];
+		path[0] = one.getLocation().toOSString();
+		path[1] = two.getLocation().toOSString();
+
+		Map<String, String> definedSymbols = new HashMap<>();
+		definedSymbols.put("__GNUC__", "5");
+		definedSymbols.put("__GNUC_MINOR__", "0");
+
+		IScannerInfo scannerInfo = new ExtendedScannerInfo(definedSymbols, path, new String[] {}, null);
+		FileContent reader = FileContent.create(base);
+		initializeScanner(reader, ParserLanguage.CPP, ParserMode.COMPLETE_PARSE, scannerInfo);
+		validateIdentifier("identifier");
+		validateIdentifier("identifier2");
+		validateIdentifier("identifier3");
+		validateIdentifier("identifier4");
+		validateIdentifier("identifier5");
+	}
+
+	// void foo() {
+	//   __has_include;
+	// }
+	public void testHasIncludeProblem() throws Exception {
+		IFile base = importFile("test.cpp", getAboveComment());
+		IScannerInfo scannerInfo = new ExtendedScannerInfo(Collections.EMPTY_MAP, null, new String[] {}, null);
+		FileContent reader = FileContent.create(base);
+		initializeScanner(reader, ParserLanguage.CPP, ParserMode.COMPLETE_PARSE, scannerInfo);
+		fullyTokenize();
+		validateProblem(0, IProblem.PREPROCESSOR_INVALID_USE_OUTSIDE_PREPROCESSOR_DIRECTIVE, "__has_include");
+	}
+
 	// #include <inc/test.h>
 	public void testRelativeIncludes_243170() throws Exception {
 		String content = getAboveComment();

core/org.eclipse.cdt.core/META-INF/MANIFEST.MF

@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: %pluginName
 Bundle-SymbolicName: org.eclipse.cdt.core; singleton:=true
-Bundle-Version: 7.0.0.qualifier
+Bundle-Version: 7.1.0.qualifier
 Bundle-Activator: org.eclipse.cdt.core.CCorePlugin
 Bundle-Vendor: %providerName
 Bundle-Localization: plugin

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/GNUScannerExtensionConfiguration.java

@@ -51,6 +51,8 @@ public abstract class GNUScannerExtensionConfiguration extends AbstractScannerEx
 		addMacro("__builtin_va_arg(ap,type)", "*(typeof(type) *)ap");
 		addMacro("__builtin_types_compatible_p(x,y)", "__builtin_types_compatible_p(sizeof(x),sizeof(y))");
 		addMacro("__offsetof__(x)", "(x)");
+		addMacro("__has_include", "");
+		addMacro("__has_include_next", "");
 
 		addPreprocessorKeyword(Keywords.cINCLUDE_NEXT, IPreprocessorDirective.ppInclude_next);
 		addPreprocessorKeyword(Keywords.cIMPORT, IPreprocessorDirective.ppImport);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IProblem.java

@@ -346,6 +346,11 @@ public interface IProblem {
 	 */
 	public final static int PREPROCESSOR_MULTIPLE_USER_DEFINED_SUFFIXES_IN_CONCATENATION = PREPROCESSOR_RELATED | 0x010;
 
+	/**
+	 * @since 7.1
+	 */
+	public final static int PREPROCESSOR_INVALID_USE_OUTSIDE_PREPROCESSOR_DIRECTIVE = PREPROCESSOR_RELATED | 0x011;
+
 	/*
 	 * Syntax error, detected by the parser.
 	 */

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/Keywords.java

@@ -314,6 +314,14 @@ public class Keywords {
 	public static final char[] cDEFINED = "defined".toCharArray();
 	/** @since 5.11 */
 	public static final char[] c__HAS_FEATURE = "__has_feature".toCharArray();
+	/**
+	 * @since 7.1
+	 */
+	public static final char[] c__HAS_INCLUDE = "__has_include".toCharArray();
+	/**
+	 * @since 7.1
+	 */
+	public static final char[] c__HAS_INCLUDE_NEXT = "__has_include_next".toCharArray();
 	/** @since 5.2*/
 	public static final char[] _Pragma = "_Pragma".toCharArray();
 	public static final char[] cVA_ARGS = "__VA_ARGS__".toCharArray();

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ASTProblem.java

@@ -97,6 +97,8 @@ public class ASTProblem extends ASTNode implements IASTProblem {
 				ParserMessages.getString("ScannerProblemFactory.error.scanner.floatWithBadPrefix")); //$NON-NLS-1$
 		errorMessages.put(Integer.valueOf(PREPROCESSOR_MULTIPLE_USER_DEFINED_SUFFIXES_IN_CONCATENATION), ParserMessages
 				.getString("ScannerProblemFactory.error.preproc.multipleUserDefinedLiteralSuffixesOnStringLiteral")); //$NON-NLS-1$
+		errorMessages.put(Integer.valueOf(PREPROCESSOR_INVALID_USE_OUTSIDE_PREPROCESSOR_DIRECTIVE),
+				ParserMessages.getString("ScannerProblemFactory.error.preproc.invalidUsageOutsidePreprocDirective")); //$NON-NLS-1$
 		errorMessages.put(Integer.valueOf(SYNTAX_ERROR),
 				ParserMessages.getString("ParserProblemFactory.error.syntax.syntaxError")); //$NON-NLS-1$
 		errorMessages.put(Integer.valueOf(MISSING_SEMICOLON),

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ParserMessages.properties

@@ -30,6 +30,7 @@ ScannerProblemFactory.error.preproc.macroPasting=Invalid use of macro pasting in
 ScannerProblemFactory.error.preproc.missingRParen=missing '')'' in parameter list of macro: {0}
 ScannerProblemFactory.error.preproc.invalidVaArgs=__VA_ARGS__ can only appear in the expansion of a variadic macro\u0020
 ScannerProblemFactory.error.preproc.multipleUserDefinedLiteralSuffixesOnStringLiteral=Multiple user-defined suffixes found when concatenating string literals
+ScannerProblemFactory.error.preproc.invalidUsageOutsidePreprocDirective="{0}" can only appear in a preprocessor directive
 
 ScannerProblemFactory.error.scanner.invalidEscapeChar=Invalid escape character encountered\u0020
 ScannerProblemFactory.error.scanner.unboundedString=Unbounded string encountered\u0020

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java

@@ -92,6 +92,8 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
 	public static final int tNOSPACE = IToken.FIRST_RESERVED_PREPROCESSOR + 4;
 	public static final int tMACRO_PARAMETER = IToken.FIRST_RESERVED_PREPROCESSOR + 5;
 	public static final int t__HAS_FEATURE = IToken.FIRST_RESERVED_PREPROCESSOR + 6;
+	public static final int t__HAS_INCLUDE = IToken.FIRST_RESERVED_PREPROCESSOR + 7;
+	public static final int t__HAS_INCLUDE_NEXT = IToken.FIRST_RESERVED_PREPROCESSOR + 8;
 
 	private static final int ORIGIN_PREPROCESSOR_DIRECTIVE = OffsetLimitReachedException.ORIGIN_PREPROCESSOR_DIRECTIVE;
 	private static final int ORIGIN_INACTIVE_CODE = OffsetLimitReachedException.ORIGIN_INACTIVE_CODE;
@@ -1323,6 +1325,11 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
 
 	}
 
+	InternalFileContent findInclusion(final String includeDirective, final boolean quoteInclude,
+			final boolean includeNext) {
+		return findInclusion(includeDirective, quoteInclude, includeNext, getCurrentFilename(), createCodeReaderTester);
+	}
+
 	private <T> T findInclusion(final String includeDirective, final boolean quoteInclude, final boolean includeNext,
 			final String currentFile, final IIncludeFileTester<T> tester) {
 		T reader = null;
@@ -1808,7 +1815,7 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
 		fLocationMap.skippedFile(fLocationMap.getSequenceNumberForOffset(offset), fi);
 	}
 
-	private char[] extractHeaderName(final char[] image, final char startDelim, final char endDelim, int[] offsets) {
+	char[] extractHeaderName(final char[] image, final char startDelim, final char endDelim, int[] offsets) {
 		char[] headerName;
 		int start = 0;
 		int length = image.length;
@@ -2124,9 +2131,6 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
 	 * If applicable the macro is expanded and the resulting tokens are put onto a new context.
 	 * @param identifier the token where macro expansion may occur.
 	 * @param lexer the input for the expansion.
-	 * @param stopAtNewline whether or not tokens to be read are limited to the current line.
-	 * @param isPPCondition whether the expansion is inside of a preprocessor condition. This
-	 * implies a specific handling for the defined token.
 	 */
 	private boolean expandMacro(final Token identifier, Lexer lexer, int options, boolean withinExpansion)
 			throws OffsetLimitReachedException {
@@ -2141,7 +2145,24 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
 				identifier.setType(t__HAS_FEATURE);
 				return false;
 			}
+			if (CharArrayUtils.equals(name, Keywords.c__HAS_INCLUDE)) {
+				identifier.setType(t__HAS_INCLUDE);
+				return false;
+			}
+			if (CharArrayUtils.equals(name, Keywords.c__HAS_INCLUDE_NEXT)) {
+				identifier.setType(t__HAS_INCLUDE_NEXT);
+				return false;
+			}
 		}
+
+		// These should not expand as macros and are not allowed outside #if, #ifdef
+		if (CharArrayUtils.equals(name, Keywords.c__HAS_INCLUDE)
+				|| CharArrayUtils.equals(name, Keywords.c__HAS_INCLUDE_NEXT)) {
+			handleProblem(IProblem.PREPROCESSOR_INVALID_USE_OUTSIDE_PREPROCESSOR_DIRECTIVE, name,
+					identifier.getOffset(), identifier.getEndOffset());
+			return false;
+		}
+
 		PreprocessorMacro macro = fMacroDictionary.get(name);
 		if (macro == null) {
 			if (reportSignificant && (options & IGNORE_UNDEFINED_SIGNIFICANT_MACROS) == 0)

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java

@@ -265,6 +265,10 @@ public class ExpressionEvaluator {
 			return handleDefined();
 		case CPreprocessor.t__HAS_FEATURE:
 			return handleHasFeature();
+		case CPreprocessor.t__HAS_INCLUDE:
+			return handleHasInclude(false);
+		case CPreprocessor.t__HAS_INCLUDE_NEXT:
+			return handleHasInclude(true);
 		case IToken.tLPAREN:
 			consume();
 			long r1 = expression();
@@ -350,6 +354,51 @@ public class ExpressionEvaluator {
 		return supported ? 1 : 0;
 	}
 
+	private long handleHasInclude(boolean isIncludeNext) throws EvalException {
+		consume(); // '__has_include'
+		if (LA() != IToken.tLPAREN) {
+			throw new EvalException(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, null);
+		}
+		consume(); // opening parenthesis
+
+		// We don't have a tSYSTEM_HEADER_NAME or tQUOTED_HEADER_NAME token type here, these only get created when
+		// executing an include directive. In the case of a tSTRING, we only have to extract the header name from the
+		// quotes (extractHeaderName) but for system header, we have to reassemble the string by concatenating the
+		// tokens.
+		// A possibly more elegant solution could be to generate tSYSTEM_HEADER_NAME or tQUOTED_HEADER_NAME (during
+		// internalFetchToken?) but this is more ambitious and intrusive (more refactoring), especially considering
+		// various cases like macro expansion.
+		String headerName;
+		boolean quoteInclude = false;
+		if (LA() == IToken.tLT) {
+			headerName = ""; //$NON-NLS-1$
+			while (fTokens.getType() != IToken.tGT) {
+				if (fTokens.getType() == IToken.tEND_OF_INPUT) {
+					throw new EvalException(IProblem.SCANNER_UNBOUNDED_STRING, null);
+				}
+				headerName += fTokens.getImage();
+				consume();
+			}
+			consume();
+			headerName += ">"; //$NON-NLS-1$
+			headerName = new String(fPreprocessor.extractHeaderName(headerName.toCharArray(), '<', '>', new int[2]));
+		} else if (LA() == IToken.tSTRING) {
+			quoteInclude = true;
+			headerName = new String(fPreprocessor.extractHeaderName(fTokens.getCharImage(), '"', '"', new int[2]));
+			consume();
+		} else {
+			throw new EvalException(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, null);
+		}
+
+		if (LA() != IToken.tRPAREN) {
+			throw new EvalException(IProblem.SCANNER_MISSING_R_PAREN, null);
+		}
+		consume(); // closing parenthesis
+
+		boolean includePathExists = fPreprocessor.findInclusion(headerName, quoteInclude, isIncludeNext) != null;
+		return includePathExists ? 1 : 0;
+	}
+
 	private int LA() {
 		return fTokens.getType();
 	}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/MacroExpander.java

@@ -376,6 +376,12 @@ public class MacroExpander {
 					t.setType(CPreprocessor.t__HAS_FEATURE);
 					result.append(t);
 					protect = true;
+				} else if (protectIntrinsics && Arrays.equals(image, Keywords.c__HAS_INCLUDE)) {
+					t.setType(CPreprocessor.t__HAS_INCLUDE);
+					result.append(t);
+				} else if (protectIntrinsics && Arrays.equals(image, Keywords.c__HAS_INCLUDE_NEXT)) {
+					t.setType(CPreprocessor.t__HAS_INCLUDE_NEXT);
+					result.append(t);
 				} else if (macro == null || (macro.isFunctionStyle() && !input.findLParenthesis())) {
 					// Tricky: Don't mark function-style macros if you don't find the left parenthesis
 					if (fReportMacros != null) {

core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/contentassist2/CompletionTest_MacroRef_NoPrefix.java

@@ -33,7 +33,8 @@ public class CompletionTest_MacroRef_NoPrefix extends CompletionProposalsBaseTes
 	private final String[] expectedResults = { "AMacro(x)", "DEBUG", "XMacro(x, y)", "__CDT_PARSER__", "__COUNTER__",
 			"__DATE__", "__FILE__", "__LINE__", "__STDC__", "__TIME__", "__builtin_va_arg(ap, type)",
 			"__builtin_offsetof(T, m)", "__builtin_types_compatible_p(x, y)", "__complex__", "__cplusplus",
-			"__extension__", "__imag__", "__null", "__offsetof__(x)", "__real__", "__stdcall", "__thread", };
+			"__extension__", "__has_include", "__has_include_next", "__imag__", "__null", "__offsetof__(x)", "__real__",
+			"__stdcall", "__thread", };
 
 	public CompletionTest_MacroRef_NoPrefix(String name) {
 		super(name);

core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/contentassist2/CompletionTests_PlainC.java

@@ -343,8 +343,8 @@ public class CompletionTests_PlainC extends AbstractContentAssistTest {
 		final String[] expected = { "AMacro(x)", "DEBUG", "XMacro(x, y)", "__CDT_PARSER__", "__COUNTER__", "__DATE__",
 				"__FILE__", "__LINE__", "__STDC_HOSTED__", "__STDC_VERSION__", "__STDC__", "__TIME__",
 				"__builtin_va_arg(ap, type)", "__builtin_offsetof(T, m)", "__builtin_types_compatible_p(x, y)",
-				"__complex__", "__extension__", "__imag__", "__null", "__offsetof__(x)", "__real__", "__stdcall",
+				"__complex__", "__extension__", "__has_include", "__has_include_next", "__imag__", "__null",
-				"__thread", };
+				"__offsetof__(x)", "__real__", "__stdcall", "__thread", };
 		assertCompletionResults(expected);
 	}