diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPTests.java index 98a18631337..0f6998b7483 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPTests.java @@ -12653,4 +12653,23 @@ public class AST2CPPTests extends AST2CPPTestBase { public void testNestedNamespaceDefinition_490359() throws Exception { parseAndCheckBindings(); } + + // constexpr auto l_a = 0b01'100'100'100; + // constexpr auto l_b = 1'123'456; + // constexpr auto l_c = 0x1000'1000; + // constexpr auto l_d = 0111'1000; + // auto v_a = 1'123'456ul; + // auto v_b = 1'123'456ull; + // auto v_c = 0xAABB'CCDDll; + public void testLiteralDecimalSeparators_519062() throws Exception { + BindingAssertionHelper helper = getAssertionHelper(); + helper.assertVariableValue("l_a", 804); + helper.assertVariableValue("l_b", 1123456); + helper.assertVariableValue("l_c", 268439552); + helper.assertVariableValue("l_d", 299520); + + helper.assertVariableType("v_a", CPPBasicType.UNSIGNED_LONG); + helper.assertVariableType("v_b", CPPBasicType.UNSIGNED_LONG_LONG); + helper.assertVariableType("v_c", CPPBasicType.LONG_LONG); + } } diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java index ee83ac3d8b8..1e0b056dd9b 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java @@ -34,6 +34,7 @@ public class LexerTests extends BaseTestCase { NO_MINMAX.fSupportMinAndMax= false; SLASH_PERCENT.fSupportSlashPercentComments= true; CPP_OPTIONS.fSupportRawStringLiterals= true; + CPP_OPTIONS.fSupportDigitSeparators= true; } static String TRIGRAPH_REPLACES_CHARS= "#^[]|{}~\\"; @@ -452,7 +453,15 @@ public class LexerTests extends BaseTestCase { eof(); } } - + + public void testNumberSeparator() throws Exception { + String n = "123'456"; + + init(n, CPP_OPTIONS); + integer(n); + eof(); + } + public void testCharLiteral() throws Exception { String lit= "'abc0123\\'\".:; \\\\'"; init(lit); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java index aed5441aecb..92d0d67b958 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java @@ -120,7 +120,12 @@ public abstract class AbstractScannerExtensionConfiguration implements IScannerE public boolean supportUserDefinedLiterals() { return false; } - + + @Override + public boolean supportDigitSeparators() { + return false; + } + @Override public CharArrayIntMap getAdditionalPreprocessorKeywords() { return fAddPreprocessorKeywords; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/GNUScannerExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/GNUScannerExtensionConfiguration.java index 127c7c70f90..51ea0b6390c 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/GNUScannerExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/GNUScannerExtensionConfiguration.java @@ -98,6 +98,12 @@ public abstract class GNUScannerExtensionConfiguration extends AbstractScannerEx return false; } + + @Override + public boolean supportDigitSeparators() { + return false; + } + /** * @deprecated simply derive from this class and use {@link #addMacro(String, String)} to * add additional macros. diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java index d14e4d558d7..29c60a41ea5 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java @@ -120,4 +120,10 @@ public interface IScannerExtensionConfiguration { * @since 5.11 */ public boolean supportUserDefinedLiterals(); + + /** + * Support for C++14 digit separators in integer literals + * @since 6.5 + */ + public boolean supportDigitSeparators(); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java index b13394c93d4..76f6fee4fa5 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java @@ -197,4 +197,9 @@ public class GPPScannerExtensionConfiguration extends GNUScannerExtensionConfigu public boolean supportUserDefinedLiterals() { return true; } + + @Override + public boolean supportDigitSeparators() { + return true; + } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java index e1843b984ae..6823438ff37 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java @@ -511,7 +511,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx * 0 * octal-literal octal-digit */ - while (isOctal(c) && i < value.length) { + while (isOctalOrSeparator(c) && i < value.length) { c = value[++i]; } break; @@ -539,7 +539,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx * decimal-literal digit */ c = value[i]; - while (Character.isDigit(c) && i < value.length) { + while (isDigitOrSeparator(c) && i < value.length) { c = value[++i]; } @@ -570,7 +570,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx */ private static int afterDecimalPoint(char[] value, int i) { char c = value[++i]; - while (Character.isDigit(c) && i < value.length) { + while (isDigitOrSeparator(c) && i < value.length) { c = value[++i]; } @@ -592,7 +592,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx c = value[++i]; } - while (Character.isDigit(c) && i < value.length) { + while (isDigitOrSeparator(c) && i < value.length) { c = value[++i]; } // If there were no digits following the 'e' then we have @@ -606,7 +606,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx char c = value[++i]; if (c == '1' || c == '0') { - while (c == '1' || c == '0' && i < value.length) { + while (c == '1' || c == '0' || c == '\'' && i < value.length) { c = value[i++]; } if (Character.isDigit(c)) { @@ -615,7 +615,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx } else if (c == '.') { // no such thing as binary floating point c = value[++i]; - while (Character.isDigit(c) && i < value.length) { + while (isDigitOrSeparator(c) && i < value.length) { c = value[i++]; } } @@ -633,8 +633,8 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx * hexadecimal-literal hexadecimal-digit */ char c = value[++i]; - if (isHexDigit(c)) { - while (isHexDigit(c) && i < value.length) { + if (isHexDigitOrSeparator(c)) { + while (isHexDigitOrSeparator(c) && i < value.length) { c = value[++i]; } if (c == '.') { @@ -654,8 +654,8 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx private static int hexFloatAfterDecimal(char[] value, int i) { // 0xHHH. char c = value[++i]; - if (isHexDigit(c)) { - while (isHexDigit(c) && i < value.length) { + if (isHexDigitOrSeparator(c)) { + while (isHexDigitOrSeparator(c) && i < value.length) { c = value[++i]; } @@ -683,7 +683,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx } if (Character.isDigit(c)) { - while (Character.isDigit(c) && i < value.length) { + while (isDigitOrSeparator(c) && i < value.length) { c = value[++i]; } } else { @@ -692,13 +692,17 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx return i; } - private static boolean isHexDigit(char c) { - c |= 0x20; - return ((c <= 'f' && c >= 'a') || (c <= '9' && c >= '0')); + private static boolean isHexDigitOrSeparator(char c) { + char lc = Character.toLowerCase(c); + return (lc <= 'f' && lc >= 'a') || (c <= '9' && c >= '0') || (c == '\''); } - private static boolean isOctal(final char c) { - return c >= '0' && c <= '7'; + private static boolean isOctalOrSeparator(final char c) { + return (c >= '0' && c <= '7') || (c == '\''); + } + + private static boolean isDigitOrSeparator(final char c) { + return Character.isDigit(c) || (c == '\''); } /** diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java index 00fe6a9fb52..ab99ea55161 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java @@ -323,6 +323,7 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { fLexOptions.fSupportUTFLiterals = configuration.supportUTFLiterals(); fLexOptions.fSupportRawStringLiterals = configuration.supportRawStringLiterals(); fLexOptions.fSupportUserDefinedLiterals = configuration.supportUserDefinedLiterals(); + fLexOptions.fSupportDigitSeparators = configuration.supportDigitSeparators(); if (info instanceof ExtendedScannerInfo) fLexOptions.fIncludeExportPatterns = ((ExtendedScannerInfo) info).getIncludeExportPatterns(); fLocationMap= new LocationMap(fLexOptions); @@ -1070,6 +1071,12 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { switch (image[pos]) { case '0': case'1': continue; + case '\'': + if (fLexOptions.fSupportDigitSeparators) { + continue; + } else { + break loop; + } case 'e': case 'E': case '.': handleProblem(IProblem.SCANNER_FLOAT_WITH_BAD_PREFIX, "0b".toCharArray(), number.getOffset(), number.getEndOffset()); //$NON-NLS-1$ @@ -1108,6 +1115,14 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { hasDot= true; continue; + // C++14 literal separator + case '\'': + if (fLexOptions.fSupportDigitSeparators) { + continue; + } else { + break loop; + } + // check for exponent or hex digit case 'E': case 'e': if (isHex && !hasExponent) { diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java index cbc301d16db..57fd47bb48d 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java @@ -439,6 +439,10 @@ public class ExpressionEvaluator { int i = from; if (from != to) { for (; i < to; i++) { + if (tokenImage[i] == '\'') { + continue; + } + int digit = getDigit(tokenImage[i]); if (digit >= base) { break; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java index 99b1872cd86..011f1df64f5 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java @@ -59,6 +59,7 @@ final public class Lexer implements ITokenSequence { public boolean fSupportUTFLiterals= true; public boolean fSupportRawStringLiterals= false; public boolean fSupportUserDefinedLiterals = false; + public boolean fSupportDigitSeparators = false; public IncludeExportPatterns fIncludeExportPatterns; @Override @@ -1033,7 +1034,14 @@ final public class Lexer implements ITokenSequence { break; } break; - + + // C++ 14 literal separator + case '\'': + if (!fOptions.fSupportDigitSeparators) { + isPartOfNumber = false; + } + break; + case END_OF_INPUT: if (fSupportContentAssist) { throw new OffsetLimitReachedException(ORIGIN_LEXER,