1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-09-10 03:53:21 +02:00

Bug 519062: add support for digit separators

Change-Id: I6fa990c76395dcc6f9b0e5e05707cff03a34b8db
Signed-off-by: Vlad Ivanov <vlad@ivanov.email>
This commit is contained in:
Vlad Ivanov 2018-04-19 13:36:26 +03:00 committed by Nathan Ridge
parent b9bac6912e
commit 7d9e0b0ddd
10 changed files with 100 additions and 19 deletions

View file

@ -12653,4 +12653,23 @@ public class AST2CPPTests extends AST2CPPTestBase {
public void testNestedNamespaceDefinition_490359() throws Exception {
parseAndCheckBindings();
}
// constexpr auto l_a = 0b01'100'100'100;
// constexpr auto l_b = 1'123'456;
// constexpr auto l_c = 0x1000'1000;
// constexpr auto l_d = 0111'1000;
// auto v_a = 1'123'456ul;
// auto v_b = 1'123'456ull;
// auto v_c = 0xAABB'CCDDll;
public void testLiteralDecimalSeparators_519062() throws Exception {
BindingAssertionHelper helper = getAssertionHelper();
helper.assertVariableValue("l_a", 804);
helper.assertVariableValue("l_b", 1123456);
helper.assertVariableValue("l_c", 268439552);
helper.assertVariableValue("l_d", 299520);
helper.assertVariableType("v_a", CPPBasicType.UNSIGNED_LONG);
helper.assertVariableType("v_b", CPPBasicType.UNSIGNED_LONG_LONG);
helper.assertVariableType("v_c", CPPBasicType.LONG_LONG);
}
}

View file

@ -34,6 +34,7 @@ public class LexerTests extends BaseTestCase {
NO_MINMAX.fSupportMinAndMax= false;
SLASH_PERCENT.fSupportSlashPercentComments= true;
CPP_OPTIONS.fSupportRawStringLiterals= true;
CPP_OPTIONS.fSupportDigitSeparators= true;
}
static String TRIGRAPH_REPLACES_CHARS= "#^[]|{}~\\";
@ -452,7 +453,15 @@ public class LexerTests extends BaseTestCase {
eof();
}
}
public void testNumberSeparator() throws Exception {
String n = "123'456";
init(n, CPP_OPTIONS);
integer(n);
eof();
}
public void testCharLiteral() throws Exception {
String lit= "'abc0123\\'\".:; \\\\'";
init(lit);

View file

@ -120,7 +120,12 @@ public abstract class AbstractScannerExtensionConfiguration implements IScannerE
public boolean supportUserDefinedLiterals() {
return false;
}
@Override
public boolean supportDigitSeparators() {
return false;
}
@Override
public CharArrayIntMap getAdditionalPreprocessorKeywords() {
return fAddPreprocessorKeywords;

View file

@ -98,6 +98,12 @@ public abstract class GNUScannerExtensionConfiguration extends AbstractScannerEx
return false;
}
@Override
public boolean supportDigitSeparators() {
return false;
}
/**
* @deprecated simply derive from this class and use {@link #addMacro(String, String)} to
* add additional macros.

View file

@ -120,4 +120,10 @@ public interface IScannerExtensionConfiguration {
* @since 5.11
*/
public boolean supportUserDefinedLiterals();
/**
* Support for C++14 digit separators in integer literals
* @since 6.5
*/
public boolean supportDigitSeparators();
}

View file

@ -197,4 +197,9 @@ public class GPPScannerExtensionConfiguration extends GNUScannerExtensionConfigu
public boolean supportUserDefinedLiterals() {
return true;
}
@Override
public boolean supportDigitSeparators() {
return true;
}
}

View file

@ -511,7 +511,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
* 0
* octal-literal octal-digit
*/
while (isOctal(c) && i < value.length) {
while (isOctalOrSeparator(c) && i < value.length) {
c = value[++i];
}
break;
@ -539,7 +539,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
* decimal-literal digit
*/
c = value[i];
while (Character.isDigit(c) && i < value.length) {
while (isDigitOrSeparator(c) && i < value.length) {
c = value[++i];
}
@ -570,7 +570,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
*/
private static int afterDecimalPoint(char[] value, int i) {
char c = value[++i];
while (Character.isDigit(c) && i < value.length) {
while (isDigitOrSeparator(c) && i < value.length) {
c = value[++i];
}
@ -592,7 +592,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
c = value[++i];
}
while (Character.isDigit(c) && i < value.length) {
while (isDigitOrSeparator(c) && i < value.length) {
c = value[++i];
}
// If there were no digits following the 'e' then we have
@ -606,7 +606,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
char c = value[++i];
if (c == '1' || c == '0') {
while (c == '1' || c == '0' && i < value.length) {
while (c == '1' || c == '0' || c == '\'' && i < value.length) {
c = value[i++];
}
if (Character.isDigit(c)) {
@ -615,7 +615,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
} else if (c == '.') {
// no such thing as binary floating point
c = value[++i];
while (Character.isDigit(c) && i < value.length) {
while (isDigitOrSeparator(c) && i < value.length) {
c = value[i++];
}
}
@ -633,8 +633,8 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
* hexadecimal-literal hexadecimal-digit
*/
char c = value[++i];
if (isHexDigit(c)) {
while (isHexDigit(c) && i < value.length) {
if (isHexDigitOrSeparator(c)) {
while (isHexDigitOrSeparator(c) && i < value.length) {
c = value[++i];
}
if (c == '.') {
@ -654,8 +654,8 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
private static int hexFloatAfterDecimal(char[] value, int i) {
// 0xHHH.
char c = value[++i];
if (isHexDigit(c)) {
while (isHexDigit(c) && i < value.length) {
if (isHexDigitOrSeparator(c)) {
while (isHexDigitOrSeparator(c) && i < value.length) {
c = value[++i];
}
@ -683,7 +683,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
}
if (Character.isDigit(c)) {
while (Character.isDigit(c) && i < value.length) {
while (isDigitOrSeparator(c) && i < value.length) {
c = value[++i];
}
} else {
@ -692,13 +692,17 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
return i;
}
private static boolean isHexDigit(char c) {
c |= 0x20;
return ((c <= 'f' && c >= 'a') || (c <= '9' && c >= '0'));
private static boolean isHexDigitOrSeparator(char c) {
char lc = Character.toLowerCase(c);
return (lc <= 'f' && lc >= 'a') || (c <= '9' && c >= '0') || (c == '\'');
}
private static boolean isOctal(final char c) {
return c >= '0' && c <= '7';
private static boolean isOctalOrSeparator(final char c) {
return (c >= '0' && c <= '7') || (c == '\'');
}
private static boolean isDigitOrSeparator(final char c) {
return Character.isDigit(c) || (c == '\'');
}
/**

View file

@ -323,6 +323,7 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
fLexOptions.fSupportUTFLiterals = configuration.supportUTFLiterals();
fLexOptions.fSupportRawStringLiterals = configuration.supportRawStringLiterals();
fLexOptions.fSupportUserDefinedLiterals = configuration.supportUserDefinedLiterals();
fLexOptions.fSupportDigitSeparators = configuration.supportDigitSeparators();
if (info instanceof ExtendedScannerInfo)
fLexOptions.fIncludeExportPatterns = ((ExtendedScannerInfo) info).getIncludeExportPatterns();
fLocationMap= new LocationMap(fLexOptions);
@ -1070,6 +1071,12 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
switch (image[pos]) {
case '0': case'1':
continue;
case '\'':
if (fLexOptions.fSupportDigitSeparators) {
continue;
} else {
break loop;
}
case 'e': case 'E':
case '.':
handleProblem(IProblem.SCANNER_FLOAT_WITH_BAD_PREFIX, "0b".toCharArray(), number.getOffset(), number.getEndOffset()); //$NON-NLS-1$
@ -1108,6 +1115,14 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
hasDot= true;
continue;
// C++14 literal separator
case '\'':
if (fLexOptions.fSupportDigitSeparators) {
continue;
} else {
break loop;
}
// check for exponent or hex digit
case 'E': case 'e':
if (isHex && !hasExponent) {

View file

@ -439,6 +439,10 @@ public class ExpressionEvaluator {
int i = from;
if (from != to) {
for (; i < to; i++) {
if (tokenImage[i] == '\'') {
continue;
}
int digit = getDigit(tokenImage[i]);
if (digit >= base) {
break;

View file

@ -59,6 +59,7 @@ final public class Lexer implements ITokenSequence {
public boolean fSupportUTFLiterals= true;
public boolean fSupportRawStringLiterals= false;
public boolean fSupportUserDefinedLiterals = false;
public boolean fSupportDigitSeparators = false;
public IncludeExportPatterns fIncludeExportPatterns;
@Override
@ -1033,7 +1034,14 @@ final public class Lexer implements ITokenSequence {
break;
}
break;
// C++ 14 literal separator
case '\'':
if (!fOptions.fSupportDigitSeparators) {
isPartOfNumber = false;
}
break;
case END_OF_INPUT:
if (fSupportContentAssist) {
throw new OffsetLimitReachedException(ORIGIN_LEXER,