1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-04-21 21:52:10 +02:00

Add c++20 character types

This commit is contained in:
Igor V. Kovalenko 2023-03-18 11:11:19 +03:00 committed by Jonah Graham
parent e8f17beeb5
commit ad15e0e804
26 changed files with 172 additions and 14 deletions

View file

@ -26,15 +26,27 @@ public abstract class AST2CPPTestBase extends AST2TestBase {
}
protected IASTTranslationUnit parseAndCheckBindings(String code) throws Exception {
return parseAndCheckBindings(code, CPP);
return parseAndCheckBindings(code, ScannerKind.STD);
}
protected IASTTranslationUnit parseAndCheckBindings(String code, ScannerKind scannerKind) throws Exception {
return parseAndCheckBindings(code, CPP, scannerKind);
}
protected IASTTranslationUnit parseAndCheckBindings() throws Exception {
return parseAndCheckBindings(ScannerKind.STD);
}
protected IASTTranslationUnit parseAndCheckBindings(ScannerKind scannerKind) throws Exception {
String code = getAboveComment();
return parseAndCheckBindings(code);
return parseAndCheckBindings(code, scannerKind);
}
protected BindingAssertionHelper getAssertionHelper() throws ParserException, IOException {
return getAssertionHelper(CPP);
return getAssertionHelper(ScannerKind.GNU);
}
protected BindingAssertionHelper getAssertionHelper(ScannerKind scannerKind) throws ParserException, IOException {
return getAssertionHelper(CPP, scannerKind);
}
}

View file

@ -157,6 +157,7 @@ public abstract class AST2TestBase extends SemanticTestBase {
private static Map<String, String> getStdCpp20Map() {
Map<String, String> map = getStdMap();
map.put("__cpp_impl_three_way_comparison", "201907L");
map.put("__cpp_char8_t", "201811L");
return map;
}

View file

@ -0,0 +1,46 @@
/*******************************************************************************
* This program and the accompanying materials
* are made available under the terms of the Eclipse Public License 2.0
* which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*******************************************************************************/
package org.eclipse.cdt.core.parser.tests.ast2.cxx20;
import org.eclipse.cdt.core.dom.ast.IBasicType.Kind;
import org.eclipse.cdt.core.dom.ast.IType;
import org.eclipse.cdt.core.parser.ParserLanguage;
import org.eclipse.cdt.core.parser.tests.ast2.AST2CPPTestBase;
import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPBasicType;
import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPPointerType;
import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPQualifierType;
import junit.framework.TestSuite;
public class CXX20CharacterTypes extends AST2CPPTestBase {
public static TestSuite suite() {
return suite(CXX20CharacterTypes.class);
}
// char test_char;
// char8_t test_char8_t;
public void testCxx20CharacterTypes() throws Exception {
parseAndCheckBindings(ScannerKind.STDCPP20);
}
// auto u8 = u8"";
public void testStringLiteralPrefixChar8t() throws Exception {
IType eChar8 = createStringType(Kind.eChar8);
BindingAssertionHelper bh = getAssertionHelper(ParserLanguage.CPP, ScannerKind.STDCPP20);
assertType(bh.assertNonProblem("u8 = ", 2), eChar8);
}
protected IType createStringType(Kind kind) {
IType type = new CPPBasicType(kind, 0);
type = new CPPQualifierType(type, true, false);
return new CPPPointerType(type);
}
}

View file

@ -688,6 +688,9 @@ public class ASTStringUtil {
case IASTSimpleDeclSpecifier.t_wchar_t:
buffer.append(Keywords.WCHAR_T).append(' ');
break;
case IASTSimpleDeclSpecifier.t_char8_t:
buffer.append(Keywords.CHAR8_T).append(' ');
break;
case IASTSimpleDeclSpecifier.t_char16_t:
buffer.append(Keywords.CHAR16_T).append(' ');
break;

View file

@ -733,6 +733,14 @@ public class ASTSignatureUtil {
result.append(Keywords.WCHAR_T);
needSpace = true;
break;
case IASTSimpleDeclSpecifier.t_char8_t:
if (needSpace) {
result.append(SPACE);
needSpace = false;
}
result.append(Keywords.CHAR8_T);
needSpace = true;
break;
case IASTSimpleDeclSpecifier.t_char16_t:
if (needSpace) {
result.append(SPACE);

View file

@ -451,6 +451,11 @@ public class ASTTypeUtil {
result.append(SPACE);
result.append(Keywords.WCHAR_T);
break;
case eChar8:
if (needSpace)
result.append(SPACE);
result.append(Keywords.CHAR8_T);
break;
case eChar16:
if (needSpace)
result.append(SPACE);

View file

@ -140,6 +140,12 @@ public interface IASTSimpleDeclSpecifier extends IASTDeclSpecifier {
*/
public static final int t_decltype_auto = 18;
/**
* <code>char8_t c;</code>
* @since 8.1
*/
public static final int t_char8_t = 19;
/**
* @since 5.1
*/

View file

@ -37,7 +37,9 @@ public interface IBasicType extends IType {
/** @since 5.10 */
eDecimal64,
/** @since 5.10 */
eDecimal128;
eDecimal128,
/** @since 8.1 */
eChar8;
}
/** @since 5.2 */

View file

@ -258,6 +258,11 @@ public interface ICPPNodeFactory extends INodeFactory {
@Override
public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep);
/**
* @since 8.1
*/
public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep, boolean useChar8Type);
/**
* @since 6.5
*/

View file

@ -119,6 +119,8 @@ public interface IToken {
int t_case = 62;
int t_catch = 63;
int t_char = 64;
/** @since 8.1 */
int t_char8_t = 8102;
/** @since 5.2 */
int t_char16_t = 5202;
/** @since 5.2 */

View file

@ -49,6 +49,8 @@ public class Keywords {
public static final String CASE = "case";
public static final String CATCH = "catch";
public static final String CHAR = "char";
/** @since 8.1 */
public static final String CHAR8_T = "char8_t";
/** @since 5.2 */
public static final String CHAR16_T = "char16_t";
/** @since 5.2 */
@ -157,6 +159,8 @@ public class Keywords {
public static final char[] cCASE = "case".toCharArray();
public static final char[] cCATCH = "catch".toCharArray();
public static final char[] cCHAR = "char".toCharArray();
/** @since 8.1 */
public static final char[] cCHAR8_T = CHAR8_T.toCharArray();
/** @since 5.2 */
public static final char[] cCHAR16_T = CHAR16_T.toCharArray();
/** @since 5.2 */
@ -395,6 +399,10 @@ public class Keywords {
ckeywords.put(Keywords.c_IMAGINARY, IToken.t__Imaginary);
}
private static void addCpp20(CharArrayIntMap cppkeywords) {
cppkeywords.put(Keywords.cCHAR8_T, IToken.t_char8_t);
}
private static void addCpp(CharArrayIntMap cppkeywords) {
cppkeywords.put(Keywords.cALIGNAS, IToken.t_alignas);
cppkeywords.put(Keywords.cALIGNOF, IToken.t_alignof);

View file

@ -2787,6 +2787,7 @@ public abstract class AbstractGNUSourceCodeParser implements ISourceCodeParser {
case IToken.tCOLONCOLON:
case IToken.t_void:
case IToken.t_char:
case IToken.t_char8_t:
case IToken.t_char16_t:
case IToken.t_char32_t:
case IToken.t_wchar_t:

View file

@ -136,6 +136,7 @@ public abstract class ArithmeticConversion {
switch (kind) {
case eBoolean:
case eChar:
case eChar8:
case eChar16:
case eChar32:
case eInt:
@ -240,6 +241,7 @@ public abstract class ArithmeticConversion {
switch (kind) {
case eBoolean:
case eChar:
case eChar8:
case eWChar:
case eChar16:
return createBasicType(Kind.eInt, domain.getModifier());
@ -384,6 +386,7 @@ public abstract class ArithmeticConversion {
return n < (Integer.MAX_VALUE + 1L) * 2;
case eChar:
case eChar8:
return 0 <= n && n <= 0xFF;
case eChar16:
@ -415,6 +418,7 @@ public abstract class ArithmeticConversion {
private static long getApproximateSize(IBasicType type) {
switch (type.getKind()) {
case eChar:
case eChar8:
return 1;
case eWChar:
return 2;

View file

@ -230,6 +230,7 @@ public class SizeofCalculator {
case eBoolean:
return sizeof_bool;
case eChar:
case eChar8:
return SIZE_1;
case eInt:
return type.isShort() ? sizeof_short

View file

@ -84,17 +84,23 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
private ICPPEvaluation fEvaluation;
private IBinding fUserDefinedLiteralOperator;
private IASTImplicitName[] fImplicitNames;
private final boolean fUseChar8Type;
public CPPASTLiteralExpression(int kind, char[] value) {
this(kind, value, CharArrayUtils.EMPTY);
this(kind, value, CharArrayUtils.EMPTY, false);
}
public CPPASTLiteralExpression(int kind, char[] value, char[] numericCompilerSuffixes) {
public CPPASTLiteralExpression(int kind, char[] value, boolean useChar8Type) {
this(kind, value, CharArrayUtils.EMPTY, useChar8Type);
}
public CPPASTLiteralExpression(int kind, char[] value, char[] numericCompilerSuffixes, boolean useChar8Type) {
fKind = kind;
fSuffix = getSuffix(kind, value, CharArrayUtils.EMPTY);
fLiteral = getLiteral(value, fSuffix);
fNumericCompilerSuffixes = (numericCompilerSuffixes == null) ? CharArrayUtils.EMPTY : numericCompilerSuffixes;
fStringLiteralSize = -1;
fUseChar8Type = useChar8Type;
}
private CPPASTLiteralExpression(CPPASTLiteralExpression other) {
@ -106,6 +112,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
fEvaluation = other.fEvaluation;
fUserDefinedLiteralOperator = other.fUserDefinedLiteralOperator;
fImplicitNames = other.fImplicitNames;
fUseChar8Type = other.fUseChar8Type;
}
@Override
@ -436,11 +443,15 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
case 'U':
return Kind.eChar32;
case 'u':
// Bug 526724 u8 should result in Kind.eChar
if (fLiteral[1] != '8') {
if (fLiteral[1] == '8') {
if (fUseChar8Type) {
return Kind.eChar8;
} else {
return Kind.eChar;
}
} else {
return Kind.eChar16;
}
//$FALL-THROUGH$
default:
return Kind.eChar;
}

View file

@ -85,6 +85,8 @@ public class CPPASTSimpleDeclSpecifier extends CPPASTBaseDeclSpecifier implement
return t_char;
case eWChar:
return t_wchar_t;
case eChar8:
return t_char8_t;
case eChar16:
return t_char16_t;
case eChar32:

View file

@ -127,6 +127,8 @@ public class CPPBasicType implements ICPPBasicType, ISerializableType {
return Kind.eChar;
case IASTSimpleDeclSpecifier.t_wchar_t:
return Kind.eWChar;
case IASTSimpleDeclSpecifier.t_char8_t:
return Kind.eChar8;
case IASTSimpleDeclSpecifier.t_char16_t:
return Kind.eChar16;
case IASTSimpleDeclSpecifier.t_char32_t:
@ -342,6 +344,7 @@ public class CPPBasicType implements ICPPBasicType, ISerializableType {
case eBoolean:
return t_bool;
case eChar:
case eChar8:
case eChar16:
case eChar32:
return t_char;

View file

@ -584,12 +584,17 @@ public class CPPNodeFactory extends NodeFactory implements ICPPNodeFactory {
@Override
public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep) {
return new CPPASTLiteralExpression(kind, rep.toCharArray());
return new CPPASTLiteralExpression(kind, rep.toCharArray(), false);
}
@Override
public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep, boolean useChar8Type) {
return new CPPASTLiteralExpression(kind, rep.toCharArray(), useChar8Type);
}
@Override
public ICPPASTLiteralExpression newLiteralExpression(int kind, String rep, char[] numericCompilerSuffixes) {
return new CPPASTLiteralExpression(kind, rep.toCharArray(), numericCompilerSuffixes);
return new CPPASTLiteralExpression(kind, rep.toCharArray(), numericCompilerSuffixes, false);
}
@Override

View file

@ -209,6 +209,7 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser {
private final boolean supportUserDefinedLiterals;
private final boolean supportGCCStyleDesignators;
private final boolean supportFoldExpression;
private final boolean supportChar8TypeLiterals;
private final IIndex index;
protected ICPPASTTranslationUnit translationUnit;
@ -247,6 +248,7 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser {
fContextSensitiveTokens = createContextSensitiveTokenMap(config);
additionalNumericalSuffixes = scanner.getAdditionalNumericLiteralSuffixes();
supportFoldExpression = true;
supportChar8TypeLiterals = scanner.getMacroDefinitions().containsKey("__cpp_char8_t"); //$NON-NLS-1$
}
@Override
@ -672,6 +674,7 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser {
// Start of a postfix expression
case IToken.t_typename:
case IToken.t_char:
case IToken.t_char8_t:
case IToken.t_char16_t:
case IToken.t_char32_t:
case IToken.t_wchar_t:
@ -2060,6 +2063,7 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser {
// simple-type-specifier braced-init-list
case IToken.t_typename:
case IToken.t_char:
case IToken.t_char8_t:
case IToken.t_char16_t:
case IToken.t_char32_t:
case IToken.t_wchar_t:
@ -2253,7 +2257,8 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser {
case IToken.tUTF32CHAR:
case IToken.tUSER_DEFINED_CHAR_LITERAL:
t = consume();
literalExpr = getNodeFactory().newLiteralExpression(IASTLiteralExpression.lk_char_constant, t.getImage());
literalExpr = getNodeFactory().newLiteralExpression(IASTLiteralExpression.lk_char_constant, t.getImage(),
supportChar8TypeLiterals);
literalExprWithRange = setRange(literalExpr, t.getOffset(), t.getEndOffset());
break;
case IToken.t_false:
@ -2349,7 +2354,7 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser {
}
IToken t = consume();
ICPPASTLiteralExpression r = getNodeFactory().newLiteralExpression(IASTLiteralExpression.lk_string_literal,
t.getImage());
t.getImage(), supportChar8TypeLiterals);
return setRange(r, t.getOffset(), t.getEndOffset());
}
@ -3726,6 +3731,13 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser {
encounteredRawType = true;
endOffset = consume().getEndOffset();
break;
case IToken.t_char8_t:
if (encounteredTypename)
break declSpecifiers;
simpleType = IASTSimpleDeclSpecifier.t_char8_t;
encounteredRawType = true;
endOffset = consume().getEndOffset();
break;
case IToken.t_char16_t:
if (encounteredTypename)
break declSpecifiers;

View file

@ -261,7 +261,7 @@ class AggregateInitialization {
ICPPBasicType t = getBasicTypeFromArray(target);
if (t != null) {
Kind k = t.getKind();
return k == Kind.eChar || k == Kind.eChar16 || k == Kind.eChar32 || k == Kind.eWChar;
return k == Kind.eChar || k == Kind.eChar8 || k == Kind.eChar16 || k == Kind.eChar32 || k == Kind.eWChar;
}
return false;
}

View file

@ -617,6 +617,7 @@ class BuiltinOperators {
return true;
case eBoolean:
case eChar:
case eChar8:
case eChar16:
case eChar32:
case eInt:
@ -637,6 +638,7 @@ class BuiltinOperators {
switch (kind) {
case eBoolean:
case eChar:
case eChar8:
case eChar16:
case eChar32:
case eDouble:
@ -664,6 +666,7 @@ class BuiltinOperators {
switch (kind) {
case eBoolean:
case eChar:
case eChar8:
case eChar16:
case eChar32:
case eInt:

View file

@ -1014,6 +1014,7 @@ public class Conversions {
case eChar:
case eBoolean:
case eWChar:
case eChar8:
case eChar16:
case eUnspecified: // treat unspecified as int
if (!basicTgt.isUnsigned()) {
@ -1405,6 +1406,13 @@ public class Conversions {
if (num instanceof Byte)
return num;
return Byte.valueOf(num.byteValue());
case eChar8: {
int intVal = num.intValue();
int maskedVal = intVal & 0xFF;
if (maskedVal == intVal && num instanceof Integer)
return num;
return Integer.valueOf(maskedVal);
}
case eChar16:
int intVal = num.intValue();
int maskedVal = intVal & 0xFFFF;

View file

@ -178,6 +178,7 @@ public class EvalTypeId extends CPPDependentEvaluation {
case eFloat128:
case eNullPtr:
case eChar:
case eChar8:
case eChar16:
case eChar32:
case eWChar:

View file

@ -247,6 +247,7 @@ public class TemplateArgumentDeduction {
case eInt128:
case eBoolean:
case eChar:
case eChar8:
case eChar16:
case eChar32:
case eWChar:

View file

@ -117,6 +117,10 @@ public class DeclSpecWriter extends NodeWriter {
if (isCpp)
return Keywords.WCHAR_T;
break;
case IASTSimpleDeclSpecifier.t_char8_t:
if (isCpp)
return Keywords.CHAR8_T;
break;
case IASTSimpleDeclSpecifier.t_char16_t:
if (isCpp)
return Keywords.CHAR16_T;

View file

@ -100,6 +100,7 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
private static final char[] ONE = "1".toCharArray(); //$NON-NLS-1$
private static final char[] CPP_IMPL_THREE_WAY_COMPARISON = "__cpp_impl_three_way_comparison".toCharArray(); //$NON-NLS-1$
private static final char[] CPP_CHAR8_T = "__cpp_char8_t".toCharArray(); //$NON-NLS-1$
// Standard built-ins
private static final ObjectStyleMacro __CDT_PARSER__ = new ObjectStyleMacro("__CDT_PARSER__".toCharArray(), //$NON-NLS-1$
@ -353,6 +354,9 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
fIncludeSearchPath = configureIncludeSearchPath(new File(contextPath).getParentFile(), info);
setupMacroDictionary(configuration, info, language);
if (fMacroDictionary.containsKey(CPP_CHAR8_T)) {
fKeywords.put(Keywords.cCHAR8_T, IToken.t_char8_t);
}
if (fMacroDictionary.containsKey(CPP_IMPL_THREE_WAY_COMPARISON)) {
fLexOptions.fSupportThreeWayComparisonOperator = true;
}