Bug 519062: add support for digit separators

Change-Id: I6fa990c76395dcc6f9b0e5e05707cff03a34b8db
Signed-off-by: Vlad Ivanov <vlad@ivanov.email>

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPTests.java

@@ -12653,4 +12653,23 @@ public class AST2CPPTests extends AST2CPPTestBase {
 	public void testNestedNamespaceDefinition_490359() throws Exception {
 		parseAndCheckBindings();
 	}
+
+	//	constexpr auto l_a = 0b01'100'100'100;
+	//	constexpr auto l_b = 1'123'456;
+	//	constexpr auto l_c = 0x1000'1000;
+	//	constexpr auto l_d = 0111'1000;
+	//	auto v_a = 1'123'456ul;
+	//	auto v_b = 1'123'456ull;
+	//	auto v_c = 0xAABB'CCDDll;
+	public void testLiteralDecimalSeparators_519062() throws Exception {
+		BindingAssertionHelper helper = getAssertionHelper();
+		helper.assertVariableValue("l_a", 804);
+		helper.assertVariableValue("l_b", 1123456);
+		helper.assertVariableValue("l_c", 268439552);
+		helper.assertVariableValue("l_d", 299520);
+
+		helper.assertVariableType("v_a", CPPBasicType.UNSIGNED_LONG);
+		helper.assertVariableType("v_b", CPPBasicType.UNSIGNED_LONG_LONG);
+		helper.assertVariableType("v_c", CPPBasicType.LONG_LONG);
+	}
 }

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java

@@ -34,6 +34,7 @@ public class LexerTests extends BaseTestCase {
 		NO_MINMAX.fSupportMinAndMax= false;
 		SLASH_PERCENT.fSupportSlashPercentComments= true;
 		CPP_OPTIONS.fSupportRawStringLiterals= true;
+		CPP_OPTIONS.fSupportDigitSeparators= true;
 	}
 	
 	static String TRIGRAPH_REPLACES_CHARS= "#^[]|{}~\\";
@@ -453,6 +454,14 @@ public class LexerTests extends BaseTestCase {
 		}
 	}
 
+	public void testNumberSeparator() throws Exception {
+		String n = "123'456";
+
+		init(n, CPP_OPTIONS);
+		integer(n);
+		eof();
+	}
+
 	public void testCharLiteral() throws Exception {
 		String lit= "'abc0123\\'\".:; \\\\'";
 		init(lit);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java

@@ -121,6 +121,11 @@ public abstract class AbstractScannerExtensionConfiguration implements IScannerE
 		return false;
 	}
 
+	@Override
+	public boolean supportDigitSeparators() {
+		return false;
+	}
+
 	@Override
 	public CharArrayIntMap getAdditionalPreprocessorKeywords() {
 		return fAddPreprocessorKeywords;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/GNUScannerExtensionConfiguration.java

@@ -98,6 +98,12 @@ public abstract class GNUScannerExtensionConfiguration extends AbstractScannerEx
     	return false;
     }
     
+
+	@Override
+	public boolean supportDigitSeparators() {
+		return false;
+	}
+
 	/**
 	 * @deprecated simply derive from this class and use {@link #addMacro(String, String)} to
 	 * add additional macros.

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java

@@ -120,4 +120,10 @@ public interface IScannerExtensionConfiguration {
 	 * @since 5.11
 	 */
 	public boolean supportUserDefinedLiterals();
+
+	/**
+	 * Support for C++14 digit separators in integer literals
+	 * @since 6.5
+	 */
+	public boolean supportDigitSeparators();
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java

@@ -197,4 +197,9 @@ public class GPPScannerExtensionConfiguration extends GNUScannerExtensionConfigu
 	public boolean supportUserDefinedLiterals() {
 		return true;
 	}
+
+	@Override
+	public boolean supportDigitSeparators() {
+		return true;
+	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java

@@ -511,7 +511,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 				*   0
 				*   octal-literal octal-digit
 				*/
-				while (isOctal(c) && i < value.length) {
+				while (isOctalOrSeparator(c) && i < value.length) {
 					c = value[++i];
 				}
 				break;
@@ -539,7 +539,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 			*    decimal-literal digit
 			*/
 			c = value[i];
-			while (Character.isDigit(c) && i < value.length) {
+			while (isDigitOrSeparator(c) && i < value.length) {
 				c = value[++i];
 			}
 
@@ -570,7 +570,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 	 */
 	private static int afterDecimalPoint(char[] value, int i) {
 		char c = value[++i];
-		while (Character.isDigit(c) && i < value.length) {
+		while (isDigitOrSeparator(c) && i < value.length) {
 			c = value[++i];
 		}
 
@@ -592,7 +592,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 			c = value[++i];
 		}
 
-		while (Character.isDigit(c) && i < value.length) {
+		while (isDigitOrSeparator(c) && i < value.length) {
 			c = value[++i];
 		}
 		// If there were no digits following the 'e' then we have
@@ -606,7 +606,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 		char c = value[++i];
 
 		if (c == '1' || c == '0') {
-			while (c == '1' || c == '0' && i < value.length) {
+			while (c == '1' || c == '0' || c == '\'' && i < value.length) {
 				c = value[i++];
 			}
 			if (Character.isDigit(c)) {
@@ -615,7 +615,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 			} else if (c == '.') {
 				// no such thing as binary floating point
 				c = value[++i];
-				while (Character.isDigit(c) && i < value.length) {
+				while (isDigitOrSeparator(c) && i < value.length) {
 					c = value[i++];
 				}
 			}
@@ -633,8 +633,8 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 		 *   hexadecimal-literal hexadecimal-digit
 		 */
 		char c = value[++i];
-		if (isHexDigit(c)) {
+		if (isHexDigitOrSeparator(c)) {
-			while (isHexDigit(c) && i < value.length) {
+			while (isHexDigitOrSeparator(c) && i < value.length) {
 				c = value[++i];
 			}
 			if (c == '.') {
@@ -654,8 +654,8 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 	private static int hexFloatAfterDecimal(char[] value, int i) {
 		// 0xHHH.
 		char c = value[++i];
-		if (isHexDigit(c)) {
+		if (isHexDigitOrSeparator(c)) {
-			while (isHexDigit(c) && i < value.length) {
+			while (isHexDigitOrSeparator(c) && i < value.length) {
 				c = value[++i];
 			}
 
@@ -683,7 +683,7 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 		}
 
 		if (Character.isDigit(c)) {
-			while (Character.isDigit(c) && i < value.length) {
+			while (isDigitOrSeparator(c) && i < value.length) {
 				c = value[++i];
 			}
 		} else {
@@ -692,13 +692,17 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx
 		return i;
 	}
 
-	private static boolean isHexDigit(char c) {
+	private static boolean isHexDigitOrSeparator(char c) {
-		c |= 0x20;
+		char lc = Character.toLowerCase(c);
-		return ((c <= 'f' && c >= 'a') || (c <= '9' && c >= '0'));
+		return (lc <= 'f' && lc >= 'a') || (c <= '9' && c >= '0') || (c == '\'');
 	}
 
-	private static boolean isOctal(final char c) {
+	private static boolean isOctalOrSeparator(final char c) {
-		return c >= '0' && c <= '7';
+		return (c >= '0' && c <= '7') || (c == '\'');
+	}
+
+	private static boolean isDigitOrSeparator(final char c) {
+		return Character.isDigit(c) || (c == '\'');
 	}
 
 	/**

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java

@@ -323,6 +323,7 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
         fLexOptions.fSupportUTFLiterals = configuration.supportUTFLiterals();
         fLexOptions.fSupportRawStringLiterals = configuration.supportRawStringLiterals();
         fLexOptions.fSupportUserDefinedLiterals = configuration.supportUserDefinedLiterals();
+        fLexOptions.fSupportDigitSeparators = configuration.supportDigitSeparators();
         if (info instanceof ExtendedScannerInfo)
         	fLexOptions.fIncludeExportPatterns = ((ExtendedScannerInfo) info).getIncludeExportPatterns();
         fLocationMap= new LocationMap(fLexOptions);
@@ -1070,6 +1071,12 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
             	switch (image[pos]) {
                 case '0': case'1':
                 	continue;
+                case '\'':
+                	if (fLexOptions.fSupportDigitSeparators) {
+                    	continue;
+                	} else {
+                		break loop;
+                	}
                 case 'e': case 'E':
                 case '.':
                 	handleProblem(IProblem.SCANNER_FLOAT_WITH_BAD_PREFIX, "0b".toCharArray(), number.getOffset(), number.getEndOffset()); //$NON-NLS-1$
@@ -1108,6 +1115,14 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable {
                 hasDot= true;
                 continue;
 
+            // C++14 literal separator
+            case '\'':
+            	if (fLexOptions.fSupportDigitSeparators) {
+                    continue;
+            	} else {
+            		break loop;
+            	}
+
             // check for exponent or hex digit
             case 'E': case 'e':
                 if (isHex && !hasExponent) {

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java

@@ -439,6 +439,10 @@ public class ExpressionEvaluator {
 		int i = from;
 		if (from != to) {
 			for (; i < to; i++) {
+				if (tokenImage[i] == '\'') {
+					continue;
+				}
+
 				int digit = getDigit(tokenImage[i]);
 				if (digit >= base) {
 					break;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java

@@ -59,6 +59,7 @@ final public class Lexer implements ITokenSequence {
 		public boolean fSupportUTFLiterals= true;
 		public boolean fSupportRawStringLiterals= false;
 		public boolean fSupportUserDefinedLiterals = false; 
+		public boolean fSupportDigitSeparators = false;
 		public IncludeExportPatterns fIncludeExportPatterns;
 		
 		@Override
@@ -1034,6 +1035,13 @@ final public class Lexer implements ITokenSequence {
             	}
             	break;
 
+			// C++ 14 literal separator
+			case '\'':
+				if (!fOptions.fSupportDigitSeparators) {
+					isPartOfNumber = false;
+				}
+				break;
+
             case END_OF_INPUT:
 				if (fSupportContentAssist) {
 					throw new OffsetLimitReachedException(ORIGIN_LEXER,