From fecd602c1be263eed7c95256bf48f0df2da5ac5b Mon Sep 17 00:00:00 2001 From: Andrey Mozzhuhin Date: Sat, 4 Jan 2020 18:33:44 +0300 Subject: [PATCH] Bug 534070: Fix syntax highlight for numbers with C++14 separators Change-Id: I1e1f335dadb3fa30e035c5a61ccef1f3eed43358 Signed-off-by: Andrey Mozzhuhin --- .../cdt/ui/tests/text/CPartitionerTest.java | 11 ++++++++++ .../cdt/ui/tests/text/NumberRuleTest.java | 8 +++++++ .../ui/text/FastCPartitionScanner.java | 12 +++++++++- .../cdt/internal/ui/text/NumberRule.java | 22 +++++++++++++------ 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/CPartitionerTest.java b/core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/CPartitionerTest.java index c62cad5f33b..46e09d497be 100644 --- a/core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/CPartitionerTest.java +++ b/core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/CPartitionerTest.java @@ -1192,4 +1192,15 @@ public class CPartitionerTest extends TestCase { assertTrue(false); } } + + public void testNumberSeparators() { + try { + fDocument.replace(0, fDocument.getLength(), "1'123'456\n0x1000'1000\n0111'1000\n0xAABB'CCDD"); + ITypedRegion[] result = fDocument.computePartitioning(0, fDocument.getLength()); + TypedRegion[] expectation = { new TypedRegion(0, 43, IDocument.DEFAULT_CONTENT_TYPE) }; + checkPartitioning(expectation, result); + } catch (BadLocationException x) { + assertTrue(false); + } + } } diff --git a/core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/NumberRuleTest.java b/core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/NumberRuleTest.java index 71b19f67fd2..536626b2578 100644 --- a/core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/NumberRuleTest.java +++ b/core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/NumberRuleTest.java @@ -125,6 +125,14 @@ public class NumberRuleTest extends TestCase { assertNoNumber("+ 9"); } + public void testSeparators() { + assertNumber("1'123'456"); + assertNumber("0x1000'1000"); + assertNumber("0111'1000"); + assertNumber("0xAABB'CCDD"); + assertNoNumber("'"); + } + /** * Validate that given string is recognized as a number. * @param string diff --git a/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/FastCPartitionScanner.java b/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/FastCPartitionScanner.java index c38dd0526d1..21fcd848045 100644 --- a/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/FastCPartitionScanner.java +++ b/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/FastCPartitionScanner.java @@ -62,6 +62,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa private static final int BACKSLASH_BACKSLASH = 7; // postfix for STRING, CHARACTER private static final int RAW_STRING_R = 8; // prefix for RAW_STRING private static final int IDENT = 9; + private static final int NUMBER = 10; /** The scanner. */ private final BufferedDocumentScanner fScanner = new BufferedDocumentScanner(1000); // faster implementation @@ -311,6 +312,11 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa } case '\'': + if (fLast == NUMBER) { + // C++14 digit separator + fTokenOffset++; + break; + } fLast = NONE; // ignore fLast if (fTokenLength > 0) { return preFix(CCODE, CHARACTER, NONE, 1); @@ -373,10 +379,14 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa break; default: if ('a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_') { - fLast = IDENT; + if (fLast != NUMBER) + fLast = IDENT; fTokenOffset++; } else if ('0' <= ch && ch <= '9' && fLast == IDENT) { fTokenOffset++; + } else if (('0' <= ch && ch <= '9') || ch == '.') { + fLast = NUMBER; + fTokenOffset++; } else { consume(); } diff --git a/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/NumberRule.java b/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/NumberRule.java index 20fc9e96b4f..2cb82d0b281 100644 --- a/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/NumberRule.java +++ b/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/NumberRule.java @@ -58,7 +58,7 @@ public class NumberRule implements IRule { // hexnumber starting with [+-]?0[xX] do { ch = scanner.read(); - } while (isHexNumberPart((char) ch)); + } while (isHexDigitOrSeparator(ch)); scanner.unread(); return token; } @@ -72,12 +72,12 @@ public class NumberRule implements IRule { // need at least one digit do { ch = scanner.read(); - } while (Character.isDigit((char) ch)); + } while (isDecDigitOrSeparator(ch)); if (ch == '.' && startCh != '.') { // fraction do { ch = scanner.read(); - } while (Character.isDigit((char) ch)); + } while (isDecDigitOrSeparator(ch)); } if (ch == 'e' || ch == 'E') { // exponent @@ -85,7 +85,7 @@ public class NumberRule implements IRule { if (ch == '-' || ch == '+' || Character.isDigit((char) ch)) { do { ch = scanner.read(); - } while (Character.isDigit((char) ch)); + } while (isDecDigitOrSeparator(ch)); } } scanner.unread(); @@ -107,13 +107,21 @@ public class NumberRule implements IRule { return ch == '-' || ch == '+' || ch == '.' || Character.isDigit((char) ch); } + /** + * Checks if part of decimal number; + * @param ch Char to check. + * @return true + */ + private boolean isDecDigitOrSeparator(int ch) { + return Character.isDigit((char) ch) || (ch == '\''); + } + /** * Checks if part of hex number; * @param ch Char to check. * @return true */ - private boolean isHexNumberPart(int ch) { - return Character.isDigit((char) ch) || ch == 'a' || ch == 'b' || ch == 'c' || ch == 'd' || ch == 'e' - || ch == 'f' || ch == 'A' || ch == 'B' || ch == 'C' || ch == 'D' || ch == 'E' || ch == 'F'; + private boolean isHexDigitOrSeparator(int ch) { + return Character.isDigit((char) ch) || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') || (ch == '\''); } }