From f5942dac812c36d3e56c4f2f8b94df19d487609e Mon Sep 17 00:00:00 2001 From: Richard Eames Date: Wed, 26 Mar 2014 19:31:05 -0600 Subject: [PATCH] Bug 379684 Support for User Defined Literals This patch adds preliminary support for C++11 user defined litrals: * Syntax support * Type deduction in expressions * Template literal operators * String literal concatenation I made quite a few changes in CPPASTLiteralExpression so that it more closely follows the spec when parsing numbers. And I'd like some feedback on the changes I made to CPPSemantics with regards to template literal operators. There are also some questions I have marked in comments, which I would appreciate an answer to. Change-Id: I242ecb8f5706f516a4c891fea268a668e5e4a694 Signed-off-by: Richard Eames Reviewed-on: https://git.eclipse.org/r/24367 Reviewed-by: Sergey Prigogin Tested-by: Sergey Prigogin --- .../core/parser/tests/ast2/AST2CPPTests.java | 393 +++++++++++++++++ .../tests/scanner/PortedScannerTests.java | 34 +- .../tests/scanner/PreprocessorTests.java | 55 ++- .../tests/scanner/PreprocessorTestsBase.java | 4 + ...AbstractScannerExtensionConfiguration.java | 10 + .../GNUScannerExtensionConfiguration.java | 11 +- .../IScannerExtensionConfiguration.java | 6 + ...stractCPPParserExtensionConfiguration.java | 12 +- .../cpp/GPPScannerExtensionConfiguration.java | 10 + .../cpp/ICPPParserExtensionConfiguration.java | 7 + .../org/eclipse/cdt/core/parser/IProblem.java | 25 +- .../org/eclipse/cdt/core/parser/IToken.java | 2 + .../cdt/core/parser/util/CharArrayUtils.java | 34 +- .../internal/core/dom/parser/ASTProblem.java | 8 +- .../parser/cpp/CPPASTDeleteExpression.java | 1 - .../parser/cpp/CPPASTFunctionDeclarator.java | 2 +- .../parser/cpp/CPPASTLiteralExpression.java | 411 ++++++++++++++++-- .../parser/cpp/CPPASTSimpleDeclSpecifier.java | 2 +- .../dom/parser/cpp/GNUCPPSourceParser.java | 77 +++- .../parser/cpp/semantics/CPPSemantics.java | 162 +++++++ .../core/parser/ParserMessages.properties | 4 + .../core/parser/scanner/CPreprocessor.java | 159 +++++-- .../parser/scanner/ExpressionEvaluator.java | 87 ++-- .../internal/core/parser/scanner/Lexer.java | 42 +- .../core/parser/scanner/StringType.java | 49 ++- .../ScannerExtensionConfiguration.java | 4 + 26 files changed, 1434 insertions(+), 177 deletions(-) diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPTests.java index 89d105b5674..32a8c92df5e 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPTests.java @@ -14,6 +14,7 @@ * Thomas Corbat (IFS) * Nathan Ridge * Marc-Andre Laperle + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.core.parser.tests.ast2; @@ -54,12 +55,14 @@ import org.eclipse.cdt.core.dom.ast.IASTFunctionDefinition; import org.eclipse.cdt.core.dom.ast.IASTIdExpression; import org.eclipse.cdt.core.dom.ast.IASTImplicitName; import org.eclipse.cdt.core.dom.ast.IASTImplicitNameOwner; +import org.eclipse.cdt.core.dom.ast.IASTInitializer; import org.eclipse.cdt.core.dom.ast.IASTLabelStatement; import org.eclipse.cdt.core.dom.ast.IASTLiteralExpression; import org.eclipse.cdt.core.dom.ast.IASTName; import org.eclipse.cdt.core.dom.ast.IASTNameOwner; import org.eclipse.cdt.core.dom.ast.IASTNamedTypeSpecifier; import org.eclipse.cdt.core.dom.ast.IASTNode; +import org.eclipse.cdt.core.dom.ast.IASTProblem; import org.eclipse.cdt.core.dom.ast.IASTProblemDeclaration; import org.eclipse.cdt.core.dom.ast.IASTReturnStatement; import org.eclipse.cdt.core.dom.ast.IASTSimpleDeclSpecifier; @@ -133,6 +136,7 @@ import org.eclipse.cdt.core.dom.ast.cpp.ICPPReferenceType; import org.eclipse.cdt.core.dom.ast.cpp.ICPPUsingDeclaration; import org.eclipse.cdt.core.dom.ast.cpp.ICPPVariable; import org.eclipse.cdt.core.dom.ast.cpp.SemanticQueries; +import org.eclipse.cdt.core.parser.IProblem; import org.eclipse.cdt.core.parser.ParserLanguage; import org.eclipse.cdt.core.parser.util.CharArrayUtils; import org.eclipse.cdt.internal.core.dom.parser.SizeofCalculator; @@ -251,6 +255,21 @@ public class AST2CPPTests extends AST2TestBase { assertTrue("Expected types to be the same, but first was: '" + first.toString() + "' and second was: '" + second + "'", first.isSameType(second)); } + private void checkUserDefinedLiteralIsType(String code, String type_name) throws Exception { + IASTTranslationUnit tu = parseAndCheckBindings(code, CPP); + IASTDeclaration[] declarations = tu.getDeclarations(); + IASTDeclaration declaration = declarations[declarations.length - 1]; + + IASTInitializer init = ((IASTSimpleDeclaration) declaration).getDeclarators()[0].getInitializer(); + IType type = ((IASTExpression)((IASTEqualsInitializer) init).getInitializerClause()).getExpressionType(); + + assertEquals(type_name, type.toString()); + } + + private void checkUserDefinedLiteralIsRet(String code) throws Exception { + checkUserDefinedLiteralIsType(code, "Ret"); + } + // #define CURLOPTTYPE_OBJECTPOINT 10000 // #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number // typedef enum { @@ -10577,4 +10596,378 @@ public class AST2CPPTests extends AST2TestBase { public void testU8TokenAfterIfdef_429361() throws Exception { parseAndCheckBindings(); } + + // int operator "" _A(unsigned long long i) { return 1; } + // int operator "" _B(long double d) { return 1; } + // int operator "" _C(const char* s, unsigned int sz) { return sz; } + // int operator "" _D(const wchar_t* s, unsigned int sz) { return sz; } + // int operator "" _E(const char16_t* s, unsigned int sz) { return sz; } + // int operator "" _F(const char32_t* s, unsigned int sz) { return sz; } + // int operator "" _G(char c) { return (int)c; } + // constexpr double operator "" _km_to_miles(long double km) { return km * 0.6213; } + public void testSimpleUserDefinedLiteralOperators() throws Exception { + parseAndCheckBindings(); + } + + // int integers[] = { + // 1, + // 1U, + // 1L, + // 1LL, + // 1ULL, + // 1suff, + // 1_suff, + // 0x3003, + // 0x3003U, + // 0x3003L, + // 0x3003LL, + // 0x3003ULL, + // 0x3003suff, + // 0x3003_suff, + // 0xabcdef, + // 0xABCDEF, + // 0Xabcdef, + // 0xABCDEF, + // 0xABCDEFU, + // 0xABCDEFL, + // 0xABCDEFULL, + // 0xABCDEFsuff, + // 0xABCDEF_suff, + // 01, + // 01U, + // 01L, + // 07LL, + // 04ULL, + // 01_suff, + // 1ULL << 34, + // }; + public void testIntegerUserDefinedLiterals() throws Exception { + parseAndCheckBindings(); + } + + // double numbers[] = { + // 1f, + // 1.f, + // 1.X, + // 1.0x, + // 0x01p3, + // 0x01p3XX, + // 1._X + // }; + public void testDoublesUserDefinedLiterals() throws Exception { + parseAndCheckBindings(); + } + + // char c1 = '0'_suff; + // char c2 = '0'suff; + // char* c3 = "Hello"_suff; + // char* c4 = "Hello"suff; + public void testCharStringUserDefinedLiterals() throws Exception { + parseAndCheckBindings(); + } + + // class Ret {}; + // Ret operator "" X(unsigned long long i) { return Ret(); } + // auto test = 123X; + public void testUserDefinedLiteralOperatorTypes1() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(long double i) { return Ret(); } + // auto test = 12.3X; + public void testUserDefinedLiteralOperatorTypes2() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char* s) { return Ret(); } + // auto test = 123X; + public void testUserDefinedLiteralOperatorTypes1a() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char* s) { return Ret(); } + // auto test = 12.3X; + public void testUserDefinedLiteralOperatorTypes2a() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(unsigned long long d) { return Ret(); } + // bool operator "" X(const char* s) { return false; } + // auto test = 123X; + public void testUserDefinedLiteralOperatorTypes1b() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(long double d) { return Ret(); } + // bool operator "" X(const char* s) { return false; } + // auto test = 12.3X; + public void testUserDefinedLiteralOperatorTypes2b() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char* s, unsigned sz) { return Ret(); } + // auto test = "123"X; + public void testUserDefinedLiteralOperatorTypes3() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const wchar_t* s, unsigned sz) { return Ret(); } + // auto test = L"123"X; + public void testUserDefinedLiteralOperatorTypes3a() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char16_t* s, unsigned sz) { return Ret(); } + // auto test = u"123"X; + public void testUserDefinedLiteralOperatorTypes3b() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char32_t* s, unsigned sz) { return Ret(); } + // auto test = U"123"X; + public void testUserDefinedLiteralOperatorTypes3c() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // template Ret operator "" X() { return Ret(); } + // auto test = 123X; + public void testUserDefinedLiteralOperatorTypes4a() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // template Ret operator "" X() { return Ret(); } + // auto test = 123.123X; + public void testUserDefinedLiteralOperatorTypes4b() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char* s, unsigned sz) { return Ret(); } + // auto test = "123" "123"X; + public void testUserDefinedLiteralConcatenation1a() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char* s, unsigned sz) { return Ret(); } + // auto test = "123"X "123"; + public void testUserDefinedLiteralConcatenation1b() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char* s, unsigned sz) { return Ret(); } + // auto test = u8"123" "123"X; + public void testUserDefinedLiteralConcatenation2a() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char* s, unsigned sz) { return Ret(); } + // auto test = u8"123"X "123"; + public void testUserDefinedLiteralConcatenation2b() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char* s, unsigned sz) { return Ret(); } + // auto test = "123" u8"123"X; + public void testUserDefinedLiteralConcatenation2c() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char* s, unsigned sz) { return Ret(); } + // auto test = "123"X u8"123"; + public void testUserDefinedLiteralConcatenation2d() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const wchar_t* s, unsigned sz) { return Ret(); } + // auto test = L"123" "123"X; + public void testUserDefinedLiteralConcatenation3a() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const wchar_t* s, unsigned sz) { return Ret(); } + // auto test = L"123"X "123"; + public void testUserDefinedLiteralConcatenation3b() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const wchar_t* s, unsigned sz) { return Ret(); } + // auto test = "123" L"123"X; + public void testUserDefinedLiteralConcatenation3c() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const wchar_t* s, unsigned sz) { return Ret(); } + // auto test = "123"X L"123"; + public void testUserDefinedLiteralConcatenation3d() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char16_t* s, unsigned sz) { return Ret(); } + // auto test = u"123" "123"X; + public void testUserDefinedLiteralConcatenation4a() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char16_t* s, unsigned sz) { return Ret(); } + // auto test = u"123"X "123"; + public void testUserDefinedLiteralConcatenation4b() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char16_t* s, unsigned sz) { return Ret(); } + // auto test = "123" u"123"X; + public void testUserDefinedLiteralConcatenation4c() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char16_t* s, unsigned sz) { return Ret(); } + // auto test = "123"X u"123"; + public void testUserDefinedLiteralConcatenation4d() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char32_t* s, unsigned sz) { return Ret(); } + // auto test = U"123" "123"X; + public void testUserDefinedLiteralConcatenation5a() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char32_t* s, unsigned sz) { return Ret(); } + // auto test = U"123"X "123"; + public void testUserDefinedLiteralConcatenation5b() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char32_t* s, unsigned sz) { return Ret(); } + // auto test = "123" U"123"X; + public void testUserDefinedLiteralConcatenation5c() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char32_t* s, unsigned sz) { return Ret(); } + // auto test = "123"X U"123"; + public void testUserDefinedLiteralConcatenation5d() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char32_t* s, unsigned sz) { return Ret(); } + // auto test = "123"X U"123"X; + public void testUserDefinedLiteralConcatenation6() throws Exception { + checkUserDefinedLiteralIsRet(getAboveComment()); + } + + // class Ret {}; + // Ret operator "" X(const char* s, unsigned sz) { return Ret(); } + // Ret operator "" Y(const char* s, unsigned sz) { return Ret(); } + // auto test = "123"X "123"Y; + public void testUserDefinedLiteralBadConcat1() throws Exception { + IASTTranslationUnit tu = parse(getAboveComment(), CPP, true, false); + + IASTProblem[] problems = tu.getPreprocessorProblems(); + assertEquals(1, problems.length); + + assertEquals(IProblem.PREPROCESSOR_MULTIPLE_USER_DEFINED_SUFFIXES_IN_CONCATENATION, problems[0].getID()); + } + + // // Test name lacking a space + // int operator ""X(const char* s) { return 0; } + public void testUserDefinedLiteralNoWhiteSpace1() throws Exception { + IASTTranslationUnit tu = parse(getAboveComment(), CPP, true, false); + IASTDeclaration decl = tu.getDeclarations()[0]; + + assertTrue(decl instanceof IASTProblemDeclaration); + assertEquals(IProblem.SYNTAX_ERROR, ((IASTProblemDeclaration)decl).getProblem().getID()); + } + + // // Test literals with spaces before the suffix + // int operator "" X(const char* s) { return 0; } + // auto a = 1 X; + // auto b = 1.0 X; + // auto c1 = '1' X; + // auto c2 = L'1' X; + // auto c3 = u8'1' X; + // auto c4 = u'1' X; + // auto c5 = U'1' X; + // auto d1 = "1" X; + // auto d2 = L"1" X; + // auto d3 = u8"1" X; + // auto d4 = u"1" X; + // auto d5 = U"1" X; + // auto e1 = "1" X "2"; + // auto e2 = L"1" X "2"; + // auto e3 = u8"1" X "2"; + // auto e4 = u"1" X "2"; + // auto e5 = U"1" X "2"; + // auto d5 = U"1" X; + public void testUserDefinedLiteralNoWhiteSpace2() throws Exception { + IASTTranslationUnit tu = parse(getAboveComment(), CPP, true, false); + IASTDeclaration[] decls = tu.getDeclarations(); + + for (int i = 1; i < decls.length; i++) { + IASTDeclaration decl = decls[i]; + assertTrue(decl instanceof IASTProblemDeclaration); + assertEquals(IProblem.SYNTAX_ERROR, ((IASTProblemDeclaration)decl).getProblem().getID()); + } + } + + // class RetA {}; + // class RetB {}; + // template RetA operator "" X() { return RetA(); } + // RetB operator "" X(unsigned long long i) { return RetB(); } + // auto a = 123X; + public void testUserDefinedLiteralResolution1() throws Exception { + checkUserDefinedLiteralIsType(getAboveComment(), "RetB"); + } + + // class RetA {}; + // class RetB {}; + // template RetA operator "" X() { return RetA(); } + // RetB operator "" X(long double i) { return RetB(); } + // auto a = 123.123X; + public void testUserDefinedLiteralResolution2() throws Exception { + checkUserDefinedLiteralIsType(getAboveComment(), "RetB"); + } + + // class RetA {}; + // class RetB {}; + // template RetA operator "" X() { return RetA(); } + // RetB operator "" X(const char * c) { return RetB(); } + // auto test = 123X; + /*public void testUserDefinedLiteralResolution3() throws Exception { + // TODO(Richard Eames): Enable this test when I figure out how to + // display a problem. + BindingAssertionHelper bh = new BindingAssertionHelper(getAboveComment(), true); + // resolution is abiguous + bh.assertProblem("test", 4); + } + */ } diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PortedScannerTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PortedScannerTests.java index 4862269c852..079f67e1e93 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PortedScannerTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PortedScannerTests.java @@ -1808,23 +1808,23 @@ public class PortedScannerTests extends PreprocessorTestsBase { IASTProblem[] problems= fLocationResolver.getScannerProblems(); assertEquals(17, problems.length); int i= 0; - assertEquals(IProblem.SCANNER_BAD_OCTAL_FORMAT, problems[i].getID() ); - assertEquals(IProblem.SCANNER_BAD_DECIMAL_FORMAT, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_BAD_HEX_FORMAT, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_BAD_HEX_FORMAT, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_DIVIDE_BY_ZERO, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_MISSING_R_PAREN, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_MISSING_R_PAREN, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_ILLEGAL_IDENTIFIER, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_BAD_CONDITIONAL_EXPRESSION,problems[++i].getID() ); - assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_BAD_CHARACTER, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_MISSING_R_PAREN, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_ASSIGNMENT_NOT_ALLOWED, problems[++i].getID() ); - assertEquals(IProblem.SCANNER_BAD_OCTAL_FORMAT, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_BAD_OCTAL_FORMAT, problems[i].getID() ); + assertEquals(IProblem.SCANNER_CONSTANT_WITH_BAD_SUFFIX, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_CONSTANT_WITH_BAD_SUFFIX, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_CONSTANT_WITH_BAD_SUFFIX, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_DIVIDE_BY_ZERO, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_MISSING_R_PAREN, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_MISSING_R_PAREN, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_ILLEGAL_IDENTIFIER, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_BAD_CONDITIONAL_EXPRESSION,problems[++i].getID() ); + assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_BAD_CHARACTER, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_EXPRESSION_SYNTAX_ERROR, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_MISSING_R_PAREN, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_ASSIGNMENT_NOT_ALLOWED, problems[++i].getID() ); + assertEquals(IProblem.SCANNER_BAD_OCTAL_FORMAT, problems[++i].getID() ); } public void testExpressionEvalProblems() throws Exception { diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTests.java index 90d8d87551b..7e091d58a45 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTests.java @@ -8,6 +8,7 @@ * Contributors: * IBM - Initial API and implementation * Markus Schorn (Wind River Systems) + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.core.parser.tests.scanner; @@ -18,6 +19,7 @@ import junit.framework.TestSuite; import org.eclipse.cdt.core.parser.IProblem; import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.ParserLanguage; /** @@ -222,8 +224,22 @@ public class PreprocessorTests extends PreprocessorTestsBase { // #define tp(x,y) #x##y // tp(a, ); // tp(a,b); - public void testStringifyAndPaste() throws Exception { - initializeScanner(); + public void testStringifyAndPasteCPP() throws Exception { + initializeScanner(getAboveComment(), ParserLanguage.CPP); + validateString("a"); + validateToken(IToken.tSEMI); + + validateUserDefinedLiteralString("a", "b"); + validateToken(IToken.tSEMI); + validateEOF(); + validateProblemCount(0); + } + + // #define tp(x,y) #x##y + // tp(a, ); + // tp(a,b); + public void testStringifyAndPasteC() throws Exception { + initializeScanner(getAboveComment(), ParserLanguage.C); validateString("a"); validateToken(IToken.tSEMI); @@ -1355,13 +1371,40 @@ public class PreprocessorTests extends PreprocessorTestsBase { validateToken(IToken.tSEMI); validateEOF(); validateProblemCount(0); + } + public void testBadBinaryNumbersC() throws Exception { + String badbinary = "{0b012, 0b01b, 0b1111e01, 0b1111p10, 0b10010.10010}"; + initializeScanner(badbinary, ParserLanguage.C); + fullyTokenize(); + validateProblemCount(5); + validateProblem(0, IProblem.SCANNER_BAD_BINARY_FORMAT, null); + validateProblem(1, IProblem.SCANNER_CONSTANT_WITH_BAD_SUFFIX, "b"); + validateProblem(2, IProblem.SCANNER_FLOAT_WITH_BAD_PREFIX, "0b"); + validateProblem(3, IProblem.SCANNER_CONSTANT_WITH_BAD_SUFFIX, "p10"); + validateProblem(4, IProblem.SCANNER_FLOAT_WITH_BAD_PREFIX, "0b"); + } + + public void testBadBinaryNumbersCPP() throws Exception { + // First, third, and fift are invalid in c++11 String badbinary = "{0b012, 0b01b, 0b1111e01, 0b1111p10, 0b10010.10010}"; initializeScanner(badbinary); fullyTokenize(); - validateProblemCount(5); - for (int i = 0; i < 5; i++) { - validateProblem(i, IProblem.SCANNER_BAD_BINARY_FORMAT, null); - } + validateProblemCount(3); + validateProblem(0, IProblem.SCANNER_BAD_BINARY_FORMAT, null); + validateProblem(1, IProblem.SCANNER_FLOAT_WITH_BAD_PREFIX, "0b"); + validateProblem(2, IProblem.SCANNER_FLOAT_WITH_BAD_PREFIX, "0b"); + } + + // #if 123ASDF + // #endif + // #if 0xU + // #endif + public void testUDLInPP() throws Exception { + initializeScanner(); + validateEOF(); + validateProblemCount(2); + validateProblem(0, IProblem.SCANNER_CONSTANT_WITH_BAD_SUFFIX, "ASDF"); + validateProblem(1, IProblem.SCANNER_CONSTANT_WITH_BAD_SUFFIX, "xU"); } } diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTestsBase.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTestsBase.java index 550e1a84020..496213ff825 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTestsBase.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTestsBase.java @@ -152,6 +152,10 @@ public abstract class PreprocessorTestsBase extends BaseTestCase { validateToken(IToken.tUTF32STRING, "U\"" + expectedImage + "\""); } + protected void validateUserDefinedLiteralString(String expectedImage, String expectedSuffix) throws Exception { + validateToken(IToken.tUSER_DEFINED_STRING_LITERAL, "\"" + expectedImage + "\"" + expectedSuffix); + } + protected void validateChar(String expectedImage) throws Exception { validateToken(IToken.tCHAR, "'" + expectedImage + "'"); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java index 7cd17177aeb..85efb90adfd 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java @@ -8,6 +8,7 @@ * Contributors: * Anton Leherbauer (Wind River Systems) - initial API and implementation * Markus Schorn (Wind River Systems) + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.core.dom.parser; @@ -111,6 +112,15 @@ public abstract class AbstractScannerExtensionConfiguration implements IScannerE return false; } + /** + * {@inheritDoc} + * @since 5.7 + */ + @Override + public boolean supportUserDefinedLiterals() { + return false; + } + @Override public CharArrayIntMap getAdditionalPreprocessorKeywords() { return fAddPreprocessorKeywords; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/GNUScannerExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/GNUScannerExtensionConfiguration.java index 5a507d84fba..7e96a4166cc 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/GNUScannerExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/GNUScannerExtensionConfiguration.java @@ -10,6 +10,7 @@ * Anton Leherbauer (Wind River Systems) * Markus Schorn (Wind River Systems) * Sergey Prigogin (Google) + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.core.dom.parser; @@ -89,7 +90,15 @@ public abstract class GNUScannerExtensionConfiguration extends AbstractScannerEx public char[] supportAdditionalNumericLiteralSuffixes() { return "ij".toCharArray(); //$NON-NLS-1$ } - + + /** + * @since 5.7 + */ + @Override + public boolean supportUserDefinedLiterals() { + return false; + } + /** * @deprecated simply derive from this class and use {@link #addMacro(String, String)} to * add additional macros. diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java index 839dbbbd123..cad45af1b52 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java @@ -115,4 +115,10 @@ public interface IScannerExtensionConfiguration { * @since 5.5 */ public boolean supportRawStringLiterals(); + + /** + * Support for User Defined Literals such as 123_suffix + * @since 5.7 + */ + public boolean supportUserDefinedLiterals(); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/AbstractCPPParserExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/AbstractCPPParserExtensionConfiguration.java index 07f3913ca02..95d99199e60 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/AbstractCPPParserExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/AbstractCPPParserExtensionConfiguration.java @@ -8,6 +8,7 @@ * Contributors: * Anton Leherbauer (Wind River Systems) - initial API and implementation * Markus Schorn (Wind River Systems) + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.core.dom.parser.cpp; @@ -153,7 +154,16 @@ public abstract class AbstractCPPParserExtensionConfiguration implements ICPPPar public boolean supportFunctionStyleAssembler() { return false; } - + + /** + * {@inheritDoc} + * @since 5.7 + */ + @Override + public boolean supportUserDefinedLiterals() { + return true; + } + /* * @see org.eclipse.cdt.core.dom.parser.cpp.ICPPParserExtensionConfiguration#getBuiltinBindingsProvider() */ diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java index 77bf6e8c220..55f4e19b2d0 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java @@ -11,6 +11,7 @@ * Anton Leherbauer (Wind River Systems) * Markus Schorn (Wind River Systems) * Sergey Prigogin (Google) + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.core.dom.parser.cpp; @@ -123,4 +124,13 @@ public class GPPScannerExtensionConfiguration extends GNUScannerExtensionConfigu public boolean supportRawStringLiterals() { return true; } + + /** + * User Defined Literals + * @since 5.7 + */ + @Override + public boolean supportUserDefinedLiterals() { + return true; + } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/ICPPParserExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/ICPPParserExtensionConfiguration.java index 53779093bb4..7d6946254ab 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/ICPPParserExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/ICPPParserExtensionConfiguration.java @@ -145,6 +145,13 @@ public interface ICPPParserExtensionConfiguration { * @since 5.1 */ public boolean supportFunctionStyleAssembler(); + + /** + * Support user-defined literal expressions: + * (char_expr | string_expr | int_expr | float_expr) ud-suffix + * @since 5.7 + */ + public boolean supportUserDefinedLiterals(); /** * @deprecated use {@link #getBuiltinBindingsProvider()} instead. diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IProblem.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IProblem.java index 0f037e9ea57..ad7bb5e56bd 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IProblem.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IProblem.java @@ -7,6 +7,7 @@ * * Contributors: * John Camelon (IBM Corporation) - initial API and implementation + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.core.parser; @@ -221,7 +222,19 @@ public interface IProblem { * @since 5.1 */ public final static int SCANNER_BAD_BINARY_FORMAT = SCANNER_RELATED | 0x00F; - + + /** + * Invalid suffix on constant + * @since 5.7 + */ + public final static int SCANNER_CONSTANT_WITH_BAD_SUFFIX = SCANNER_RELATED | 0x010; + + /** + * Invalid prefix on float + * @since 5.7 + */ + public final static int SCANNER_FLOAT_WITH_BAD_PREFIX = SCANNER_RELATED | 0x011; + // Preprocessor /** * #error encountered by Preprocessor. @@ -305,12 +318,12 @@ public interface IProblem { * macro argument "..." encountered without the required ')' i.e. must be last argument if used * Required attributes: none */ - public final static int PREPROCESSOR_MISSING_RPAREN_PARMLIST = PREPROCESSOR_RELATED | 0x00C; + public final static int PREPROCESSOR_MISSING_RPAREN_PARMLIST = PREPROCESSOR_RELATED | 0x00C; /** * __VA_ARGS__ encountered in macro definition without the required '...' parameter * Required attributes: none - */ + */ public final static int PREPROCESSOR_INVALID_VA_ARGS = PREPROCESSOR_RELATED | 0x00D; /** @@ -326,6 +339,12 @@ public interface IProblem { */ public final static int PREPROCESSOR_EXCEEDS_MAXIMUM_INCLUSION_DEPTH= PREPROCESSOR_RELATED | 0x00F; + /** + * During concatentation of string literals, at least two were found with + * more than one type of UDL suffix + * @since 5.7 + */ + public final static int PREPROCESSOR_MULTIPLE_USER_DEFINED_SUFFIXES_IN_CONCATENATION = PREPROCESSOR_RELATED | 0x010; /** * Syntax error, detected by the parser. */ diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java index 4917bf17d8f..5141ac3ff13 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java @@ -189,11 +189,13 @@ public interface IToken { int tLSTRING = 131; /** @since 5.1 */ int tUTF16STRING = 5000; /** @since 5.1 */ int tUTF32STRING = 5001; + /** @since 5.7 */ int tUSER_DEFINED_STRING_LITERAL = 5700; int tCHAR = 132; int tLCHAR = 133; /** @since 5.1 */ int tUTF16CHAR = 5002; /** @since 5.1 */ int tUTF32CHAR = 5003; + /** @since 5.7 */ int tUSER_DEFINED_CHAR_LITERAL = 5701; int t__Bool = 134; int t__Complex = 135; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/util/CharArrayUtils.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/util/CharArrayUtils.java index 6307942c555..97cdc507e1c 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/util/CharArrayUtils.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/util/CharArrayUtils.java @@ -10,6 +10,7 @@ * Andrew Ferguson (Symbian) * Markus Schorn (Wind River Systems) * Sergey Prigogin (Google) + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.core.parser.util; @@ -298,18 +299,41 @@ public class CharArrayUtils { } public static final int lastIndexOf(char[] toBeFound, char[] array) { - int j = toBeFound.length - 1; - for (int i = array.length; --i >= 0;) { + return lastIndexOf(toBeFound, array, 0); + } + + /** + * @since 5.7 + */ + public static int lastIndexOf(char toBeFound, char[] array) { + return lastIndexOf(toBeFound, array, 0); + } + + /** + * @since 5.7 + */ + public static int lastIndexOf(char toBeFound, char[] array, int fromIndex) { + return lastIndexOf(new char[]{toBeFound}, array, fromIndex); + } + + /** + * @since 5.7 + */ + public static int lastIndexOf(char[] toBeFound, char[] array, int fromIndex) { + int j = toBeFound.length - 1; + for (int i = array.length; --i >= fromIndex;) { if (toBeFound[j] == array[i]) { - if (--j == -1) - return i; + if (--j == -1) { + return i; + } } else { j = toBeFound.length - 1; } } + return -1; } - + static final public char[] trim(char[] chars) { if (chars == null) return null; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ASTProblem.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ASTProblem.java index 6b04283e619..9444366e6c1 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ASTProblem.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ASTProblem.java @@ -57,7 +57,7 @@ public class ASTProblem extends ASTNode implements IASTProblem { errorMessages.put(new Integer(PREPROCESSOR_MISSING_RPAREN_PARMLIST), ParserMessages.getString("ScannerProblemFactory.error.preproc.missingRParen")); //$NON-NLS-1$ errorMessages.put(new Integer(PREPROCESSOR_INVALID_VA_ARGS), - ParserMessages.getString("ScannerProblemFactory.error.preproc.invalidVaArgs")); //$NON-NLS-1$ + ParserMessages.getString("ScannerProblemFactory.error.preproc.invalidVaArgs")); //$NON-NLS-1$ errorMessages.put(new Integer(SCANNER_INVALID_ESCAPECHAR), ParserMessages.getString("ScannerProblemFactory.error.scanner.invalidEscapeChar")); //$NON-NLS-1$ errorMessages.put(new Integer(SCANNER_UNBOUNDED_STRING), @@ -88,6 +88,12 @@ public class ASTProblem extends ASTNode implements IASTProblem { ParserMessages.getString("ScannerProblemFactory.error.scanner.unexpectedEOF")); //$NON-NLS-1$ errorMessages.put(new Integer(SCANNER_BAD_CHARACTER), ParserMessages.getString("ScannerProblemFactory.error.scanner.badCharacter")); //$NON-NLS-1$ + errorMessages.put(new Integer(SCANNER_CONSTANT_WITH_BAD_SUFFIX), + ParserMessages.getString("ScannerProblemFactory.error.scanner.constantWithBadSuffix")); //$NON-NLS-1$ + errorMessages.put(new Integer(SCANNER_FLOAT_WITH_BAD_PREFIX), + ParserMessages.getString("ScannerProblemFactory.error.scanner.floatWithBadPrefix")); //$NON-NLS-1$ + errorMessages.put(new Integer(PREPROCESSOR_MULTIPLE_USER_DEFINED_SUFFIXES_IN_CONCATENATION), + ParserMessages.getString("ScannerProblemFactory.error.preproc.multipleUserDefinedLiteralSuffixesOnStringLiteral")); //$NON-NLS-1$ errorMessages.put(new Integer(SYNTAX_ERROR), ParserMessages.getString("ParserProblemFactory.error.syntax.syntaxError")); //$NON-NLS-1$ errorMessages.put(new Integer(MISSING_SEMICOLON), diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTDeleteExpression.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTDeleteExpression.java index 40a17c2b09d..aad689caf79 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTDeleteExpression.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTDeleteExpression.java @@ -17,7 +17,6 @@ import java.util.ArrayList; import java.util.List; import org.eclipse.cdt.core.dom.ast.ASTVisitor; -import org.eclipse.cdt.core.dom.ast.IASTDeclaration; import org.eclipse.cdt.core.dom.ast.IASTExpression; import org.eclipse.cdt.core.dom.ast.IASTImplicitName; import org.eclipse.cdt.core.dom.ast.IASTNode; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTFunctionDeclarator.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTFunctionDeclarator.java index ff3fbcee596..954b903d45e 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTFunctionDeclarator.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTFunctionDeclarator.java @@ -32,7 +32,7 @@ import org.eclipse.cdt.internal.core.dom.parser.IASTAmbiguityParent; * Represents a function declarator. */ public class CPPASTFunctionDeclarator extends CPPASTDeclarator implements ICPPASTFunctionDeclarator, - IASTAmbiguityParent { + IASTAmbiguityParent { private ICPPASTParameterDeclaration[] parameters; private IASTTypeId[] typeIds = NO_EXCEPTION_SPECIFICATION; private ICPPASTExpression noexceptExpression; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java index 9a9391c2ff9..58eba65c8de 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTLiteralExpression.java @@ -8,6 +8,7 @@ * Contributors: * John Camelon (IBM) - Initial API and implementation * Markus Schorn (Wind River Systems) + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.internal.core.dom.parser.cpp; @@ -15,15 +16,21 @@ import static org.eclipse.cdt.core.dom.ast.IASTExpression.ValueCategory.LVALUE; import static org.eclipse.cdt.core.dom.ast.IASTExpression.ValueCategory.PRVALUE; import org.eclipse.cdt.core.dom.ast.ASTVisitor; +import org.eclipse.cdt.core.dom.ast.DOMException; import org.eclipse.cdt.core.dom.ast.IBasicType; import org.eclipse.cdt.core.dom.ast.IBasicType.Kind; +import org.eclipse.cdt.core.dom.ast.IBinding; import org.eclipse.cdt.core.dom.ast.IScope; +import org.eclipse.cdt.core.dom.ast.ISemanticProblem; import org.eclipse.cdt.core.dom.ast.IType; import org.eclipse.cdt.core.dom.ast.IValue; import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTLiteralExpression; +import org.eclipse.cdt.core.dom.ast.cpp.ICPPFunction; import org.eclipse.cdt.core.parser.util.CharArrayUtils; import org.eclipse.cdt.internal.core.dom.parser.ASTNode; +import org.eclipse.cdt.internal.core.dom.parser.ProblemType; import org.eclipse.cdt.internal.core.dom.parser.Value; +import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.CPPSemantics; import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.CPPVisitor; import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.EvalFixed; import org.eclipse.cdt.internal.core.parser.scanner.ExpressionEvaluator; @@ -42,6 +49,8 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx private int kind; private char[] value = CharArrayUtils.EMPTY; + private char[] suffix = CharArrayUtils.EMPTY; + private boolean isCompilerSuffix = true; private ICPPEvaluation fEvaluation; public CPPASTLiteralExpression() { @@ -51,6 +60,11 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx this.kind = kind; this.value = value; } + + public CPPASTLiteralExpression(int kind, char[] value, char[] suffix) { + this(kind, value); + this.setSuffix(suffix); + } @Override public CPPASTLiteralExpression copy() { @@ -59,8 +73,10 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx @Override public CPPASTLiteralExpression copy(CopyStyle style) { - CPPASTLiteralExpression copy = - new CPPASTLiteralExpression(kind, value == null ? null : value.clone()); + CPPASTLiteralExpression copy = new CPPASTLiteralExpression(kind, + value == null ? null : value.clone(), + suffix == null ? null : suffix.clone()); + copy.setOffsetAndLength(this); return copy(copy, style); } @@ -86,7 +102,76 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx this.value= value; } - @Override + public char[] getSuffix() { + return suffix; + } + + public void setSuffix(char[] suffix) { + this.suffix = suffix; + } + + public void calculateSuffix() { + this.calculateSuffix(CharArrayUtils.EMPTY); + } + + /** + * Returns the suffix of a user-defined literal integer or float + * @param compilerSuffixes + */ + public void calculateSuffix(char[] compilerSuffixes) { + try { + switch (kind) { + case lk_float_constant: + case lk_integer_constant: + int udOffset = (value[0] == '.' ? afterDecimalPoint(0) : integerLiteral()); + if (udOffset > 0) { + /* + * 2.14.8.1 + * "If a token matches both user-defined-literal and another literal kind, it is treated as the latter" + */ + setSuffix(CharArrayUtils.subarray(value, udOffset, -1)); + for (int i = 0; i < suffix.length; i++) { + switch (suffix[i]) { + case 'l': case 'L': + case 'u': case 'U': + case 'f': case 'F': + continue; + } + for (int j = 0; j < compilerSuffixes.length; j++) { + if (suffix[i] == compilerSuffixes[j]) { + continue; + } + } + isCompilerSuffix = false; + // Remove the suffix from the value if it's a UDL + setValue(CharArrayUtils.subarray(value, 0, udOffset)); + break; + } + } + break; + case lk_string_literal: + { + final int offset = CharArrayUtils.lastIndexOf('"', value, CharArrayUtils.indexOf('"', value) + 1); + if (offset > 0) { + setSuffix(CharArrayUtils.subarray(value, offset + 1, -1)); + } + } + break; + case lk_char_constant: + { + final int offset = CharArrayUtils.lastIndexOf('\'', value, CharArrayUtils.indexOf('\'', value) + 1); + if (offset > 0) { + setSuffix(CharArrayUtils.subarray(value, offset + 1, -1)); + } + } + break; + } + } catch (ArrayIndexOutOfBoundsException e) { + // pass + } + } + + @Override public String toString() { return new String(value); } @@ -136,9 +221,41 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx } return Value.create(length); } - - private Kind getCharType() { - switch (getValue()[0]) { + + private IType getStringType() { + if (suffix.length > 0) { + return getUserDefinedLiteralOperatorType(); + } + + IType type = new CPPBasicType(getBasicCharKind(), 0, this); + type = new CPPQualifierType(type, true, false); + return new CPPArrayType(type, getStringLiteralSize()); + } + + private IType getCharType() { + return suffix.length > 0 ? getUserDefinedLiteralOperatorType() : new CPPBasicType(getBasicCharKind(), 0, this); + } + + // 13.5.8 + private IType getUserDefinedLiteralOperatorType() { + IType ret = new ProblemType(ISemanticProblem.TYPE_UNRESOLVED_NAME); + + try { + IBinding func = CPPSemantics.findUserDefinedLiteralOperator(this); + if (func != null && func instanceof ICPPFunction) { + ret = ((ICPPFunction) func).getType().getReturnType(); + } + } catch (DOMException e) { /* ignore and return the problem type */ } + + return ret; + } + + public char[] getOperatorName() { + return CharArrayUtils.concat("operator \"\"".toCharArray(), suffix); //$NON-NLS-1$ + } + + public Kind getBasicCharKind() { + switch (value[0]) { case 'L': return Kind.eWChar; case 'u': @@ -148,21 +265,25 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx default: return Kind.eChar; } - } + } private IType classifyTypeOfFloatLiteral() { - final char[] lit= getValue(); + final char[] lit= suffix; final int len= lit.length; Kind kind= Kind.eDouble; int flags= 0; if (len > 0) { - switch (lit[len - 1]) { - case 'f': case 'F': - kind= Kind.eFloat; - break; - case 'l': case 'L': - flags |= IBasicType.IS_LONG; - break; + if (isCompilerSuffix) { + switch (lit[len - 1]) { + case 'f': case 'F': + kind= Kind.eFloat; + break; + case 'l': case 'L': + flags |= IBasicType.IS_LONG; + break; + } + } else { + return getUserDefinedLiteralOperatorType(); } } return new CPPBasicType(kind, flags, this); @@ -171,36 +292,238 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx private IType classifyTypeOfIntLiteral() { int makelong= 0; boolean unsigned= false; - - final char[] lit= getValue(); - for (int i= lit.length - 1; i >= 0; i--) { - final char c= lit[i]; - if (!(c > 'f' && c <= 'z') && !(c > 'F' && c <= 'Z')) { - break; + final char[] lit= suffix; + int flags= 0; + + if (isCompilerSuffix) { + for (int i= lit.length - 1; i >= 0; i--) { + final char c= lit[i]; + if (!(c > 'f' && c <= 'z') && !(c > 'F' && c <= 'Z')) { + break; + } + switch (c) { + case 'u': + case 'U': + unsigned = true; + break; + case 'l': + case 'L': + makelong++; + break; + } } - switch (c) { - case 'u': - case 'U': - unsigned = true; - break; - case 'l': - case 'L': - makelong++; + + if (unsigned) { + flags |= IBasicType.IS_UNSIGNED; + } + + if (makelong > 1) { + flags |= IBasicType.IS_LONG_LONG; + } else if (makelong == 1) { + flags |= IBasicType.IS_LONG; + } + } + else if (lit.length > 0) { + return getUserDefinedLiteralOperatorType(); + } + return new CPPBasicType(Kind.eInt, flags, this); + } + + private int integerLiteral() { + int i = 0; + char c = value[i++]; + + if (c == '0' && i < value.length) { + // Probably octal/hex/binary + c = value[i]; + switch ((c | 0x20)) { + case 'x': + return probablyHex(i); + case 'b': + return probablyBinary(i); + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + /* octal-literal: + * 0 + * octal-literal octal-digit + */ + while (isOctal(c) && i < value.length) { + c = value[++i]; + } break; + case '.': + return afterDecimalPoint(i); + } + /* + * If there is an 8 or 9, then we have a malformed octal + */ + if (c == '8' || c == '9') { + // eat remaining numbers + c = value[i]; + while (Character.isDigit(c) && i < value.length) { + c = value[++i]; + } + return i; } } - - int flags= 0; - if (unsigned) { - flags |= IBasicType.IS_UNSIGNED; + else if (Character.isDigit(c)) { + /* decimal-literal : + * nonzero-digit (c has to be this to get into this else) + * decimal-literal digit + */ + c = value[i]; + while (Character.isDigit(c) && i < value.length) { + c = value[++i]; + } + + if (c == '.') { + return afterDecimalPoint(i); + } else if ((c | 0x20) == 'e') { + return exponentPart(i); + } + } else { + // Somehow we got called and there wasn't a digit + // Shouldn't get here + assert false; } - if (makelong > 1) { - flags |= IBasicType.IS_LONG_LONG; - } else if (makelong == 1) { - flags |= IBasicType.IS_LONG; - } - return new CPPBasicType(Kind.eInt, flags, this); + return i; + } + + /* + * Called with the expectation that value[i] == '.' + */ + private int afterDecimalPoint(int i) { + char c = value[++i]; + while (Character.isDigit(c) && i < value.length) { + c = value[++i]; + } + + if ((c | 0x20) == 'e') { + return exponentPart(i); + } + + return i; + } + + /* + * Called with the expectation that c == 'e' + */ + private int exponentPart(int i) { + char c = value[++i]; + + // optional '+' or '-' + if (c == '+' || c == '-') { + c = value[++i]; + } + + while (Character.isDigit(c) && i < value.length) { + c = value[++i]; + } + // If there were no digits following the 'e' then we have + // D.De or .De which is a UDL on a double + + return i--; + } + + // GCC's binary constant notation + private int probablyBinary(int i) { + char c = value[++i]; + + if (c == '1' || c == '0') { + while (c == '1' || c == '0' && i < value.length) { + c = value[i++]; + } + if (Character.isDigit(c)) { + // UDL can't begin with digit, so this is a malformed binary + return -1; + } else if (c == '.') { + // no such thing as binary floating point + c = value[++i]; + while (Character.isDigit(c) && i < value.length) { + c = value[i++]; + } + } + } else { + // Here we have 0b or 0B + return i - 1; + } + return i; + } + + private int probablyHex(int i) { + /* hexadecimal-literal + * 0x hexadecimal-digit + * 0X hexadecimal-digit + * hexadecimal-literal hexadecimal-digit + */ + char c = value[++i]; + if (isHexDigit(c)) { + while (isHexDigit(c) && i < value.length) { + c = value[++i]; + } + if (c == '.') { + // Could be GCC's hex float + return hexFloatAfterDecimal(i); + } else if ((c | 0x20) == 'p') { + return hexFloatExponent(i); + } + } else { + return i - 1; + } + + return i; + } + + // Assumes value[i] == '.' + private int hexFloatAfterDecimal(int i) { + // 0xHHH. + char c = value[++i]; + if (isHexDigit(c)) { + while (isHexDigit(c) && i < value.length) { + c = value[++i]; + } + + if ((c | 0x20) == 'p') { + return hexFloatExponent(i); + } else { + // The parser is very confused at this point + // as the expression is 0x1.f + return -1; + } + } + + // Probably shouldn't be able to get here + // we have 0xHHH. + return -1; + } + + // Assumes image[i] == 'p' + private int hexFloatExponent(int i) { + // 0xHH.HH[pP][-+]?DDDD + char c = value[++i]; + + if (c == '-' || c == '+') { + c = value[++i]; + } + + if (Character.isDigit(c)) { + while (Character.isDigit(c) && i < value.length) { + c = value[++i]; + } + } else { + return i - 1; + } + return i; + } + + private boolean isHexDigit(char c) { + c |= 0x20; + return ((c <= 'f' && c >= 'a') || (c <= '9' && c >= '0')); + } + + private boolean isOctal(final char c) { + return c >= '0' && c <= '7'; } /** @@ -242,15 +565,13 @@ public class CPPASTLiteralExpression extends ASTNode implements ICPPASTLiteralEx case lk_false: return EVAL_FALSE; case lk_char_constant: - return new EvalFixed(new CPPBasicType(getCharType(), 0, this), PRVALUE, createCharValue()); - case lk_float_constant: + return new EvalFixed(getCharType(), PRVALUE, createCharValue()); + case lk_float_constant: return new EvalFixed(classifyTypeOfFloatLiteral(), PRVALUE, Value.UNKNOWN); - case lk_integer_constant: + case lk_integer_constant: return new EvalFixed(classifyTypeOfIntLiteral(), PRVALUE, createIntValue()); case lk_string_literal: - IType type = new CPPBasicType(getCharType(), 0, this); - type = new CPPQualifierType(type, true, false); - return new EvalFixed(new CPPArrayType(type, getStringLiteralSize()), LVALUE, Value.UNKNOWN); + return new EvalFixed(getStringType(), LVALUE, Value.UNKNOWN); case lk_nullptr: return EVAL_NULL_PTR; } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTSimpleDeclSpecifier.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTSimpleDeclSpecifier.java index 608e3dfd649..11520c18a17 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTSimpleDeclSpecifier.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTSimpleDeclSpecifier.java @@ -103,7 +103,7 @@ public class CPPASTSimpleDeclSpecifier extends CPPASTBaseDeclSpecifier case eVoid: return t_void; case eNullPtr: - // Null pointer type cannot be expressed wit ha simple decl specifier. + // Null pointer type cannot be expressed with a simple decl specifier. break; } return t_unspecified; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/GNUCPPSourceParser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/GNUCPPSourceParser.java index d098371ce4a..e5cdd336723 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/GNUCPPSourceParser.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/GNUCPPSourceParser.java @@ -14,6 +14,7 @@ * Andrew Ferguson (Symbian) * Sergey Prigogin (Google) * Thomas Corbat (IFS) + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.internal.core.dom.parser.cpp; @@ -159,6 +160,7 @@ import org.eclipse.cdt.internal.core.dom.parser.IASTAmbiguousStatement; import org.eclipse.cdt.internal.core.dom.parser.cpp.NameOrTemplateIDVariants.BranchPoint; import org.eclipse.cdt.internal.core.dom.parser.cpp.NameOrTemplateIDVariants.Variant; import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.CPPVisitor; +import org.eclipse.cdt.internal.core.parser.scanner.CPreprocessor; /** * This is our implementation of the IParser interface, serving as a parser for @@ -176,12 +178,14 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser { private final boolean allowCPPRestrict; private final boolean supportExtendedTemplateSyntax; private final boolean supportAutoTypeSpecifier; + private final boolean supportUserDefinedLiterals; private final IIndex index; protected ICPPASTTranslationUnit translationUnit; private int functionBodyCount; private char[] currentClassName; + private char[] additionalNumericalSuffixes; private final ICPPNodeFactory nodeFactory; private TemplateIdStrategy fTemplateParameterListStrategy; @@ -208,9 +212,11 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser { supportFunctionStyleAsm= config.supportFunctionStyleAssembler(); functionCallCanBeLValue= true; supportAutoTypeSpecifier= true; + supportUserDefinedLiterals= config.supportUserDefinedLiterals(); this.index= index; this.nodeFactory = CPPNodeFactory.getDefault(); scanner.setSplitShiftROperator(true); + additionalNumericalSuffixes = ((CPreprocessor) scanner).getAdditionalNumericLiteralSuffixes(); } @Override @@ -787,6 +793,29 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser { endOffset= consume(IToken.tGT_in_SHIFTR).getEndOffset(); op= OverloadableOperator.SHIFTR; break; + case IToken.tSTRING: // User defined literal T operator "" SUFFIX + IToken strOp = consume(); + + // Should be an empty string + if (strOp.getLength() == 2) { + endOffset = strOp.getEndOffset(); + + IToken ident = consume(IToken.tIDENTIFIER); + + // Make sure there is at least one white space + if (ident.getOffset() <= endOffset) { + break; + } + + char[] operatorName = CharArrayUtils.concat(firstToken.getCharImage(), " ".toCharArray()); //$NON-NLS-1$ + operatorName = CharArrayUtils.concat(operatorName, strOp.getCharImage()); + operatorName = CharArrayUtils.concat(operatorName, ident.getCharImage()); + + IASTName name = nodeFactory.newOperatorName(operatorName); + setRange(name, firstToken.getOffset(), ident.getEndOffset()); + return name; + } + break; default: op= OverloadableOperator.valueOf(LA(1)); if (op != null) { @@ -1731,29 +1760,46 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser { protected IASTExpression primaryExpression(CastExprCtx ctx, ITemplateIdStrategy strat) throws EndOfFileException, BacktrackException { IToken t = null; IASTLiteralExpression literalExpression = null; + IASTLiteralExpression leWithRange = null; + switch (LT(1)) { // TO DO: we need more literals... case IToken.tINTEGER: t = consume(); literalExpression = nodeFactory.newLiteralExpression(IASTLiteralExpression.lk_integer_constant, t.getImage()); - return setRange(literalExpression, t.getOffset(), t.getEndOffset()); + leWithRange = setRange(literalExpression, t.getOffset(), t.getEndOffset()); + ((CPPASTLiteralExpression) literalExpression).calculateSuffix(additionalNumericalSuffixes); + break; case IToken.tFLOATINGPT: t = consume(); literalExpression = nodeFactory.newLiteralExpression(IASTLiteralExpression.lk_float_constant, t.getImage()); - return setRange(literalExpression, t.getOffset(), t.getEndOffset()); + leWithRange = setRange(literalExpression, t.getOffset(), t.getEndOffset()); + ((CPPASTLiteralExpression) literalExpression).calculateSuffix(additionalNumericalSuffixes); + break; case IToken.tSTRING: case IToken.tLSTRING: case IToken.tUTF16STRING: case IToken.tUTF32STRING: - return stringLiteral(); + case IToken.tUSER_DEFINED_STRING_LITERAL: + leWithRange = stringLiteral(); + if (supportUserDefinedLiterals) { + ((CPPASTLiteralExpression) leWithRange).calculateSuffix(); + } + break; case IToken.tCHAR: case IToken.tLCHAR: case IToken.tUTF16CHAR: case IToken.tUTF32CHAR: - t = consume(); - literalExpression = nodeFactory.newLiteralExpression(IASTLiteralExpression.lk_char_constant, t.getImage()); - return setRange(literalExpression, t.getOffset(), t.getEndOffset()); - case IToken.t_false: + case IToken.tUSER_DEFINED_CHAR_LITERAL: + t = consume(); + literalExpression = nodeFactory.newLiteralExpression( + IASTLiteralExpression.lk_char_constant, t.getImage()); + leWithRange = setRange(literalExpression, t.getOffset(), t.getEndOffset()); + if (supportUserDefinedLiterals) { + ((CPPASTLiteralExpression) leWithRange).calculateSuffix(); + } + break; + case IToken.t_false: t = consume(); literalExpression = nodeFactory.newLiteralExpression(IASTLiteralExpression.lk_false, t.getImage()); return setRange(literalExpression, t.getOffset(), t.getEndOffset()); @@ -1805,7 +1851,21 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser { throwBacktrack(startingOffset, la.getLength()); return null; } - + + if (supportUserDefinedLiterals) { + IToken la = LA(1); + ASTNode loc = (ASTNode) leWithRange.getOriginalNode(); + if (la.getType() == IToken.tIDENTIFIER && loc != null) { + if (loc.getOffset()+loc.getLength() != la.getOffset()) { + return leWithRange; + } + IToken opName = consume(IToken.tIDENTIFIER); + ((CPPASTLiteralExpression) leWithRange).setSuffix(opName.getCharImage()); + setRange(leWithRange, loc.getOffset(), opName.getEndOffset()); + } + } + + return leWithRange; } private ICPPASTLiteralExpression stringLiteral() throws EndOfFileException, BacktrackException { @@ -1814,6 +1874,7 @@ public class GNUCPPSourceParser extends AbstractGNUSourceCodeParser { case IToken.tLSTRING: case IToken.tUTF16STRING: case IToken.tUTF32STRING: + case IToken.tUSER_DEFINED_STRING_LITERAL: break; default: throwBacktrack(LA(1)); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/CPPSemantics.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/CPPSemantics.java index 507fc570139..8caddfe5199 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/CPPSemantics.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/CPPSemantics.java @@ -14,6 +14,7 @@ * Mike Kucera (IBM) * Thomas Corbat (IFS) * Nathan Ridge + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.internal.core.dom.parser.cpp.semantics; @@ -83,6 +84,7 @@ import org.eclipse.cdt.core.dom.ast.IASTTypeId; import org.eclipse.cdt.core.dom.ast.IASTTypeIdExpression; import org.eclipse.cdt.core.dom.ast.IASTTypeIdInitializerExpression; import org.eclipse.cdt.core.dom.ast.IASTUnaryExpression; +import org.eclipse.cdt.core.dom.ast.IBasicType; import org.eclipse.cdt.core.dom.ast.IBasicType.Kind; import org.eclipse.cdt.core.dom.ast.IBinding; import org.eclipse.cdt.core.dom.ast.ICompositeType; @@ -194,17 +196,23 @@ import org.eclipse.cdt.internal.core.dom.parser.IRecursionResolvingBinding; import org.eclipse.cdt.internal.core.dom.parser.ProblemBinding; import org.eclipse.cdt.internal.core.dom.parser.Value; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTIdExpression; +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTLiteralExpression; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTName; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTNameBase; +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTTemplateId; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTTranslationUnit; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPBasicType; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPBuiltinParameter; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPCompositeBinding; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPDeferredFunction; +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPFunction; +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPFunctionInstance; +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPFunctionTemplate; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPFunctionType; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPImplicitFunction; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPNamespace; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPNamespaceScope; +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPPointerType; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPReferenceType; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPScope; import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPTemplateParameterMap; @@ -229,6 +237,7 @@ import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.Conversions.Contex import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.Conversions.UDCMode; import org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.Cost.Rank; import org.eclipse.cdt.internal.core.index.IIndexScope; +import org.eclipse.cdt.internal.core.pdom.dom.PDOMNode; import org.eclipse.core.runtime.CoreException; /** @@ -3020,6 +3029,159 @@ public class CPPSemantics { return function; } + + /** + * Given a LiteralExpression with a user-defined literal suffix, + * finds the corresponding defined operator. + * Tries to implement 2.14.8.(2-10) + * @param exp IASTLiteralExpression which has a user-defined literal suffix + * @return CPPFunction or null + * @throws DOMException + */ + public static IBinding findUserDefinedLiteralOperator(IASTLiteralExpression exp) throws DOMException { + IBinding ret = null; + /* + * 2.14.8.2 + * Let `IASTLiteralExpression exp` = L + * Let `exp.getSuffix()` = X + * Let `bindings` = S + * A user-defined-literal is treated as a call to a literal operator or + * literal operator template (13.5.8). To determine the form of this + * call for a given user-defined-literal L with ud-suffix X, the + * literal-operator-id whose literal suffix identifier is X is looked up + * in the context of L using the rules for unqualified name lookup (3.4.1). + * Let S be the set of declarations found by this lookup. + * S shall not be empty. + * + * XXX: why does findBindings return PDOMNodes? + */ + int kind = exp.getKind(); + IBinding[] bindings = findBindings(exp.getTranslationUnit().getScope(), ((CPPASTLiteralExpression) exp).getOperatorName(), false); + ICPPFunction[] funcs = new ICPPFunction[bindings.length]; + ICPPFunctionTemplate[] tplFunctions = new ICPPFunctionTemplate[bindings.length]; + LookupData data = new LookupData(((CPPASTLiteralExpression) exp).getOperatorName(), null, exp); + + int i = 0, j = 0; + for (IBinding binding : bindings) { + if (binding instanceof PDOMNode) { + continue; + } + if (binding instanceof CPPFunction || binding instanceof ICPPFunctionTemplate) { + funcs[i++] = (ICPPFunction) binding; + if (binding instanceof ICPPFunctionTemplate) { + tplFunctions[j++] = (ICPPFunctionTemplate) binding; + } + } + } + + funcs = ArrayUtil.trim(funcs); + tplFunctions = ArrayUtil.trim(tplFunctions); + + if (funcs.length == 0) { + // S shall not be empty + return new ProblemBinding(data.getLookupName(), exp, IProblemBinding.SEMANTIC_INVALID_TYPE); + } + + if (kind == IASTLiteralExpression.lk_integer_constant || kind == IASTLiteralExpression.lk_float_constant) { + if (kind == IASTLiteralExpression.lk_integer_constant) { + /* + * 2.14.8.3 + * Let `exp.getValue()` = n + * If L is a user-defined-integer-literal, let n be the literal + * without its ud-suffix. If S contains a literal operator with + * parameter type unsigned long long, then use operator "" X(n ULL) + */ + CPPBasicType t = new CPPBasicType(Kind.eInt, IBasicType.IS_UNSIGNED | IBasicType.IS_LONG_LONG, exp); + data.setFunctionArguments(false, createArgForType(exp, t)); + ret = resolveFunction(data, funcs, true); + if (ret != null && !(ret instanceof ProblemBinding)) { + return ret; + } + } else if (kind == IASTLiteralExpression.lk_float_constant) { + /* + * 2.14.8.4 + * Let `exp.getValue()` = f + * If L is a user-defined-floating-literal, let f be the literal + * without its ud-suffix. If S contains a literal operator with + * parameter type long double, then use operator "" X(f L) + */ + CPPBasicType t = new CPPBasicType(Kind.eDouble, IBasicType.IS_LONG, exp); + data.setFunctionArguments(false, createArgForType(exp, t)); + ret = resolveFunction(data, funcs, true); + if (ret != null && !(ret instanceof ProblemBinding)) { + return ret; + } + } + + /* + * 2.14.8.3 (cont.), 2.14.8.4 (cont.) + * Otherwise, S shall contain a raw literal operator or a literal + * operator template but not both. + */ + // Raw literal operator + CPPPointerType charArray = new CPPPointerType(new CPPBasicType(Kind.eChar, 0, null), true, false, false); + data = new LookupData(((CPPASTLiteralExpression) exp).getOperatorName(), null, exp); + data.setFunctionArguments(false, createArgForType(exp, charArray)); + ret = resolveFunction(data, funcs, true); + + // TODO(Richard Eames): This is a mess, but I don't know a better way to do it + + IBinding[] tplBindings = new IBinding[j]; + for (ICPPFunctionTemplate func : tplFunctions) { + // Get an instance of each template given the char literals as + // template arguments, then resolve. + CPPASTTemplateId tplName = new CPPASTTemplateId(((CPPFunctionTemplate)func).getTemplateName().copy()); + for (char c : exp.getValue()) { + tplName.addTemplateArgument(new CPPASTLiteralExpression(ICPPASTLiteralExpression.lk_char_constant, new char[]{ '\'', c, '\'' })); + } + data = new LookupData(tplName); + tplBindings[--j] = resolveFunction(data, tplFunctions, true); + } + + // Do we have valid template and non-template bindings? + if (ret != null && tplBindings.length == 1) { + if (!(ret instanceof IProblemBinding) && tplBindings[0] instanceof CPPFunctionInstance) { + // Ambiguity? It has two valid options, and the spec says it shouldn't + // TODO(Richard Eames): How do I notify the editor of an error at this point? + return new ProblemBinding(data.getLookupName(), exp, IProblemBinding.SEMANTIC_AMBIGUOUS_LOOKUP); + } + + if ((ret instanceof IProblemBinding) && tplBindings[0] instanceof CPPFunctionInstance) { + // Only the template binding is valid + ret = tplBindings[0]; + } + } + } else if (kind == IASTLiteralExpression.lk_string_literal) { + /* + * 2.14.8.5 + * If L is a user-defined-string-literal, let str be the literal + * without its ud-suffix and let len be the number of code units in + * str (i.e., its length excluding the terminating null character). + * L is treated as operater "" X(str, len) + */ + CPPPointerType strType = new CPPPointerType(new CPPBasicType(((CPPASTLiteralExpression)exp).getBasicCharKind(), 0, null), true, false, false); + CPPBasicType szType = new CPPBasicType(Kind.eInt, IBasicType.IS_UNSIGNED, null); + IASTInitializerClause[] initializer = new IASTInitializerClause[] { + createArgForType(exp, strType), + createArgForType(null, szType) + }; + data.setFunctionArguments(false, initializer); + ret = resolveFunction(data, funcs, true); + } else if (kind == IASTLiteralExpression.lk_char_constant) { + /* + * 2.14.8.6 + * If L is a user-defined-character-literal, let ch be the literal + * without its ud-suffix. S shall contain a literal operator whose + * only parameter has the type ch and the literal L is treated as a + * call operator "" X(ch) + */ + CPPBasicType t = new CPPBasicType(((CPPASTLiteralExpression)exp).getBasicCharKind(), 0, exp); + data.setFunctionArguments(false, createArgForType(exp, t)); + ret = resolveFunction(data, funcs, true); + } + + return ret; + } static ICPPFunction resolveTargetedFunction(IType targetType, CPPFunctionSet set, IASTNode point) { targetType= getNestedType(targetType, TDEF | REF | CVTYPE | PTR | MPTR); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ParserMessages.properties b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ParserMessages.properties index 62fc59c2b8d..d6bb2ed4590 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ParserMessages.properties +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ParserMessages.properties @@ -9,6 +9,7 @@ # IBM Rational Software - Initial API and implementation # Anton Leherbauer (Wind River Systems) # Markus Schorn (Wind River Systems) +# Richard Eames ############################################################################### ScannerProblemFactory.error.preproc.error=#error encountered with text: {0} @@ -25,6 +26,7 @@ ScannerProblemFactory.error.preproc.invalidDirective=Invalid preprocessor direct ScannerProblemFactory.error.preproc.macroPasting=Invalid use of macro pasting in macro: {0} ScannerProblemFactory.error.preproc.missingRParen=missing '')'' in parameter list of macro: {0} ScannerProblemFactory.error.preproc.invalidVaArgs=__VA_ARGS__ can only appear in the expansion of a variadic macro +ScannerProblemFactory.error.preproc.multipleUserDefinedLiteralSuffixesOnStringLiteral=Multiple user-defined suffixes found when concatenating string literals ScannerProblemFactory.error.scanner.invalidEscapeChar=Invalid escape character encountered ScannerProblemFactory.error.scanner.unboundedString=Unbounded string encountered @@ -41,6 +43,8 @@ ScannerProblemFactory.error.scanner.illegalIdentifier=Illegal identifier in defi ScannerProblemFactory.error.scanner.badConditionalExpression=Bad conditional expression ScannerProblemFactory.error.scanner.unexpectedEOF=Unexpected End Of File encountered ScannerProblemFactory.error.scanner.badCharacter=Bad character sequence encountered: {0} +ScannerProblemFactory.error.scanner.constantWithBadSuffix=Constant "{0}" has an invalid suffix +ScannerProblemFactory.error.scanner.floatWithBadPrefix=Floating constant "{0}" has an invalid prefix ParserProblemFactory.error.syntax.syntaxError=Syntax error ParserProblemFactory.error.syntax.missingSemicolon=Missing ';' diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java index b4fd64f5d54..872650f0f16 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java @@ -11,6 +11,7 @@ * Markus Schorn (Wind River Systems) * Sergey Prigogin (Google) * Marc-Andre Laperle (Ericsson) + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.internal.core.parser.scanner; @@ -294,6 +295,7 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { fLexOptions.fSupportSlashPercentComments= configuration.supportSlashPercentComments(); fLexOptions.fSupportUTFLiterals = configuration.supportUTFLiterals(); fLexOptions.fSupportRawStringLiterals = configuration.supportRawStringLiterals(); + fLexOptions.fSupportUserDefinedLiterals = configuration.supportUserDefinedLiterals(); if (info instanceof ExtendedScannerInfo) fLexOptions.fIncludeExportPatterns = ((ExtendedScannerInfo) info).getIncludeExportPatterns(); fLocationMap= new LocationMap(fLexOptions); @@ -386,7 +388,15 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { public ILocationResolver getLocationResolver() { return fLocationMap; } - + + /** + * @see IScannerExtensionConfiguration#supportAdditionalNumericLiteralSuffixes + * @since 5.7 + */ + public char[] getAdditionalNumericLiteralSuffixes() { + return fAdditionalNumericLiteralSuffixes; + } + private void configureKeywords(ParserLanguage language, IScannerExtensionConfiguration configuration) { Keywords.addKeywordsPreprocessor(fPPKeywords); if (language == ParserLanguage.C) { @@ -715,11 +725,12 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { */ @Override public IToken nextToken() throws EndOfFileException { - if (isCancelled) { + if (isCancelled) { throw new ParseError(ParseError.ParseErrorKind.TIMEOUT_OR_CANCELLED); } Token t1= fetchToken(); + String udl_suffix = null; final int tt1= t1.getType(); switch (tt1) { @@ -741,24 +752,33 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { case IToken.t_PRAGMA: handlePragmaOperator(t1); return nextToken(); - + case IToken.tUSER_DEFINED_STRING_LITERAL: + udl_suffix = StringType.getUserDefinedLiteralSuffix(t1); + //$FALL-THROUGH$ case IToken.tSTRING: case IToken.tLSTRING: case IToken.tUTF16STRING: case IToken.tUTF32STRING: - StringType st = StringType.fromToken(tt1); + StringType st = StringType.fromToken(t1); Token t2; StringBuilder buf= null; - int endOffset= 0; + int endOffset= t1.getEndOffset(); loop: while (true) { t2= fetchToken(); final int tt2= t2.getType(); switch (tt2) { - case IToken.tLSTRING: + case IToken.tUSER_DEFINED_STRING_LITERAL: + if (udl_suffix == null) { + udl_suffix = StringType.getUserDefinedLiteralSuffix(t2); + } else if (!udl_suffix.equals(StringType.getUserDefinedLiteralSuffix(t2))) { + handleProblem(IProblem.PREPROCESSOR_MULTIPLE_USER_DEFINED_SUFFIXES_IN_CONCATENATION, udl_suffix.toCharArray(), t2.getOffset(), endOffset); + } + //$FALL-THROUGH$ + case IToken.tLSTRING: case IToken.tSTRING: case IToken.tUTF16STRING: case IToken.tUTF32STRING: - st = StringType.max(st, StringType.fromToken(tt2)); + st = StringType.max(st, StringType.fromToken(t2)); if (buf == null) { buf= new StringBuilder(); appendStringContent(buf, t1); @@ -773,23 +793,30 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { case IToken.t_PRAGMA: handlePragmaOperator(t2); continue loop; - default: + default: break loop; } } pushbackToken(t2); if (buf != null) { char[] prefix = st.getPrefix(); - char[] image= new char[buf.length() + prefix.length + 2]; + final int imageLength = buf.length() + prefix.length + 2 + (udl_suffix == null ? 0 : udl_suffix.length()); + char[] image= new char[imageLength]; int off= -1; + int tokenType = st.getTokenValue(); for (char c : prefix) image[++off] = c; image[++off]= '"'; buf.getChars(0, buf.length(), image, ++off); - image[image.length - 1]= '"'; - t1= new TokenWithImage(st.getTokenValue(), null, t1.getOffset(), endOffset, image); + off += buf.length(); + image[off]= '"'; + if (udl_suffix != null) { + udl_suffix.getChars(0, udl_suffix.length(), image, ++off); + tokenType = IToken.tUSER_DEFINED_STRING_LITERAL; + } + t1= new TokenWithImage(tokenType, null, t1.getOffset(), endOffset, image); } break; @@ -841,7 +868,12 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { } if (length > 1) { - final int diff= image[length - 1] == '"' ? length - start - 1 : length - start; + int diff = 0; + if (t1.getType() == IToken.tUSER_DEFINED_STRING_LITERAL) { + diff = t1.getImage().lastIndexOf('"') - start; + } else { + diff= image[length - 1] == '"' ? length - start - 1 : length - start; + } if (diff > 0) { buf.append(image, start, diff); } @@ -966,6 +998,8 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { boolean isHex = false; boolean isOctal = false; boolean hasDot= false; + + boolean badSuffix = false; int pos= 0; if (image.length > 1) { @@ -1002,6 +1036,10 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { switch (image[pos]) { case '0': case'1': continue; + case 'e': case 'E': + case '.': + handleProblem(IProblem.SCANNER_FLOAT_WITH_BAD_PREFIX, "0b".toCharArray(), number.getOffset(), number.getEndOffset()); //$NON-NLS-1$ + return; default: // 0 and 1 are the only allowed digits for binary integers // No floating point, exponents etc. are allowed @@ -1069,38 +1107,71 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { break loop; } } - - // check the suffix - loop: for (; pos < image.length; pos++) { - final char c= image[pos]; - switch (c) { - case 'u': case 'U': case 'L': case 'l': - continue; - case 'f': case 'F': - if (isFloat) { - continue loop; - } - } - for (int i= 0; i < fAdditionalNumericLiteralSuffixes.length; i++) { - if (fAdditionalNumericLiteralSuffixes[i] == c) { - continue loop; - } - } - if (isBin) { - // The check for bin has to come before float, otherwise binary integers - // with float components get flagged as BAD_FLOATING_POINT - handleProblem(IProblem.SCANNER_BAD_BINARY_FORMAT, image, number.getOffset(), number.getEndOffset()); - } else if (isFloat) { - handleProblem(IProblem.SCANNER_BAD_FLOATING_POINT, image, number.getOffset(), number.getEndOffset()); - } else if (isHex) { - handleProblem(IProblem.SCANNER_BAD_HEX_FORMAT, image, number.getOffset(), number.getEndOffset()); - } else if (isOctal) { - handleProblem(IProblem.SCANNER_BAD_OCTAL_FORMAT, image, number.getOffset(), number.getEndOffset()); - } else { - handleProblem(IProblem.SCANNER_BAD_DECIMAL_FORMAT, image, number.getOffset(), number.getEndOffset()); - } - return; + if (pos < image.length) { + char c = image[pos]; + if (Character.isLetter(c) || c == '_') { + // check the suffix + final int suffixStart = pos; + loop: for (; pos < image.length; pos++) { + c= image[pos]; + switch (c) { + case 'u': case 'U': case 'L': case 'l': + continue; + case 'f': case 'F': + if (isFloat) { + continue loop; + } + + //$FALL-THROUGH$ + case 'a': case 'b': case 'c': case 'd': case 'e': case 'g': case 'h': case 'i': + case 'j': case 'k': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': + case 's': case 't': case 'v': case 'w': case 'x': case 'y': case 'z': + case 'A': case 'B': case 'C': case 'D': case 'E': case 'G': case 'H': case 'I': + case 'J': case 'K': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': + case 'S': case 'T':case 'V': case 'W': case 'X': case 'Y': case 'Z': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case '_': + if (fLexOptions.fSupportUserDefinedLiterals) { + continue loop; + } + } + for (int i= 0; i < fAdditionalNumericLiteralSuffixes.length; i++) { + if (fAdditionalNumericLiteralSuffixes[i] == c) { + continue loop; + } else { + badSuffix = true; + } + } + + if (badSuffix) { + char[] suffix = CharArrayUtils.subarray(image, suffixStart, -1); + handleProblem(IProblem.SCANNER_CONSTANT_WITH_BAD_SUFFIX, suffix, number.getOffset(), number.getEndOffset()); + return; + } + } + return; + } } + else { + return; + } + + if (isBin) { + // The check for bin has to come before float, otherwise binary integers + // with float components get flagged as BAD_FLOATING_POINT + handleProblem(IProblem.SCANNER_BAD_BINARY_FORMAT, image, number.getOffset(), number.getEndOffset()); + } else if (isFloat) { + handleProblem(IProblem.SCANNER_BAD_FLOATING_POINT, image, number.getOffset(), number.getEndOffset()); + } else if (isHex) { + handleProblem(IProblem.SCANNER_BAD_HEX_FORMAT, image, number.getOffset(), number.getEndOffset()); + } else if (isOctal) { + handleProblem(IProblem.SCANNER_BAD_OCTAL_FORMAT, image, number.getOffset(), number.getEndOffset()); + } else { + handleProblem(IProblem.SCANNER_BAD_DECIMAL_FORMAT, image, number.getOffset(), number.getEndOffset()); + } + return; + } private T findInclusion(final String includeDirective, final boolean quoteInclude, @@ -1952,7 +2023,7 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { fCurrentContext= new ScannerContext(ctx, fCurrentContext, replacement); return true; } - + @Override @SuppressWarnings({ "rawtypes", "unchecked" }) public Object getAdapter(Class adapter) { diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java index ab5810c6058..ece157f2adc 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ExpressionEvaluator.java @@ -10,6 +10,7 @@ * Markus Schorn (Wind River Systems) * Bryan Wilkinson (QNX) - https://bugs.eclipse.org/bugs/show_bug.cgi?id=151207 * Anton Leherbauer (Wind River Systems) + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.internal.core.parser.scanner; @@ -19,6 +20,7 @@ import org.eclipse.cdt.core.dom.ast.IASTName; import org.eclipse.cdt.core.parser.IProblem; import org.eclipse.cdt.core.parser.IToken; import org.eclipse.cdt.core.parser.util.CharArrayMap; +import org.eclipse.cdt.core.parser.util.CharArrayUtils; /** * Used to evaluate expressions in preprocessor directives. @@ -351,42 +353,26 @@ public class ExpressionEvaluator { // supported by GCC since 4.3 and by some other C compilers // They consist of a prefix 0b or 0B, followed by a sequence of 0 and 1 digits // see http://gcc.gnu.org/onlinedocs/gcc/Binary-constants.html - boolean isBin = false; - - boolean isHex = false; - boolean isOctal = false; - - int pos= 0; + if (image.length > 1) { if (image[0] == '0') { - switch (image[++pos]) { + switch (image[1]) { case 'b': case 'B': - isBin = true; - ++pos; - break; + return getNumber(image, 2, image.length, 2, IProblem.SCANNER_BAD_BINARY_FORMAT); + case 'x': case 'X': - isHex = true; - ++pos; - break; - case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': - isOctal = true; - ++pos; - break; + return getNumber(image, 2, image.length, 16, IProblem.SCANNER_BAD_HEX_FORMAT); + + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return getNumber(image, 1, image.length, 8, IProblem.SCANNER_BAD_OCTAL_FORMAT); } } } - if (isBin) { - return getNumber(image, 2, image.length, 2, IProblem.SCANNER_BAD_BINARY_FORMAT); - } - if (isHex) { - return getNumber(image, 2, image.length, 16, IProblem.SCANNER_BAD_HEX_FORMAT); - } - if (isOctal) { - return getNumber(image, 1, image.length, 8, IProblem.SCANNER_BAD_OCTAL_FORMAT); - } - return getNumber(image, 0, image.length, 10, IProblem.SCANNER_BAD_DECIMAL_FORMAT); + + return getNumber(image, 0, image.length, 10, IProblem.SCANNER_BAD_DECIMAL_FORMAT); } public static long getChar(char[] tokenImage, int i) throws EvalException { @@ -424,23 +410,50 @@ public class ExpressionEvaluator { } private static long getNumber(char[] tokenImage, int from, int to, int base, int problemID) throws EvalException { - if (from == to) { - throw new EvalException(problemID, tokenImage); - } - long result= 0; - int i= from; - for (; i < to; i++) { - int digit= getDigit(tokenImage[i]); - if (digit >= base) { - break; + char c; + long result = 0; + int i = from; + if (from != to) { + for (; i < to; i++) { + int digit = getDigit(tokenImage[i]); + if (digit >= base) { + break; + } + result = result*base + digit; } - result= result*base + digit; + + if (i >= to) { + return result; // return early + } + + c = tokenImage[i]; + if ('0' <= c && c <= '9') { + // A suffix cannot start with a number + throw new EvalException(problemID, tokenImage); + } else if (i == 2 && base != 10) { + // Possible that the prefix is bad. + i--; + } + + } else { + /* + * If we get in here, then the number literal is 2 characters, + * but hasn't had the last character checked if it's a suffix + */ + i = 1; } + // The rest should be a suffix + final int suffixStart = i; for (; i < to; i++) { switch (tokenImage[i]) { case 'u' : case 'l': case 'U': case 'L': break; default: + c = tokenImage[i]; + if (Character.isLetterOrDigit(c) || c == '_') { + char[] suffix = CharArrayUtils.subarray(tokenImage, suffixStart, -1); + throw new EvalException(IProblem.SCANNER_CONSTANT_WITH_BAD_SUFFIX, suffix); + } throw new EvalException(problemID, tokenImage); } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java index 810a9549f84..f7294430d17 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java @@ -9,6 +9,7 @@ * Markus Schorn - initial API and implementation * Mike Kucera (IBM) - UTF string literals * Sergey Prigogin (Google) + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.internal.core.parser.scanner; @@ -57,6 +58,7 @@ final public class Lexer implements ITokenSequence { public boolean fSupportSlashPercentComments= false; public boolean fSupportUTFLiterals= true; public boolean fSupportRawStringLiterals= false; + public boolean fSupportUserDefinedLiterals = false; public IncludeExportPatterns fIncludeExportPatterns; @Override @@ -348,7 +350,7 @@ final public class Lexer implements ITokenSequence { if (fInsideIncludeDirective) { return headerName(start, true); } - return stringLiteral(start, 1, IToken.tSTRING); + return stringLiteral(start, 1, IToken.tSTRING); case '\'': return charLiteral(start, IToken.tCHAR); @@ -732,8 +734,17 @@ final public class Lexer implements ITokenSequence { c= nextCharPhase3(); } } - - private Token stringLiteral(final int start, int length, final int tokenType) throws OffsetLimitReachedException { + + private boolean isIdentifierStart(int c) { + return Character.isLetter((char)c) || + Character.isDigit((char)c) || + Character.isUnicodeIdentifierPart(c) || + (fOptions.fSupportDollarInIdentifiers && c == '$') || + (fOptions.fSupportAtSignInIdentifiers && c == '@') || + c == '_'; + } + + private Token stringLiteral(final int start, int length, int tokenType) throws OffsetLimitReachedException { boolean escaped = false; boolean done = false; @@ -766,10 +777,17 @@ final public class Lexer implements ITokenSequence { length++; c= nextCharPhase3(); } + + if (fOptions.fSupportUserDefinedLiterals && isIdentifierStart(c)) { + Token t = identifier(start+length, 0); + tokenType = IToken.tUSER_DEFINED_STRING_LITERAL; + length += t.getLength(); + } + return newToken(tokenType, start, length); } - private Token rawStringLiteral(final int start, int length, final int tokenType) throws OffsetLimitReachedException { + private Token rawStringLiteral(final int start, int length, int tokenType) throws OffsetLimitReachedException { final int delimOffset= fOffset; int delimEndOffset = delimOffset; int offset; @@ -813,10 +831,17 @@ final public class Lexer implements ITokenSequence { fEndOffset= offset; fCharPhase3= 0; nextCharPhase3(); + + if (fOptions.fSupportUserDefinedLiterals && isIdentifierStart(fCharPhase3)) { + Token t = identifier(offset, 0); + tokenType = IToken.tUSER_DEFINED_STRING_LITERAL; + offset += t.getLength(); + } + return newToken(tokenType, start, offset - start); } - private Token charLiteral(final int start, final int tokenType) throws OffsetLimitReachedException { + private Token charLiteral(final int start, int tokenType) throws OffsetLimitReachedException { boolean escaped = false; boolean done = false; int length= tokenType == IToken.tCHAR ? 1 : 2; @@ -848,6 +873,13 @@ final public class Lexer implements ITokenSequence { length++; c= nextCharPhase3(); } + + if (fOptions.fSupportUserDefinedLiterals && isIdentifierStart(c)) { + Token t = identifier(start+length, 0); + tokenType = IToken.tUSER_DEFINED_CHAR_LITERAL; + length += t.getLength(); + } + return newToken(tokenType, start, length); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/StringType.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/StringType.java index 43982a1bab9..58a53dedd00 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/StringType.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/StringType.java @@ -7,6 +7,7 @@ * * Contributors: * Mike Kucera (IBM) - Initial API and implementation + * Richard Eames *******************************************************************************/ package org.eclipse.cdt.internal.core.parser.scanner; @@ -50,7 +51,7 @@ public enum StringType { public static StringType max(StringType st1, StringType st2) { return values()[Math.max(st1.ordinal(), st2.ordinal())]; } - + /** * Returns the StringType value for the given string literal type. * @@ -71,4 +72,50 @@ public enum StringType { throw new IllegalArgumentException(tokenVal + " is not a string token"); } } + + /** + * Returns the StringType for a given string literal token, including a + * user-defined string literal + * @see StringType#fromToken(int) + * @since 5.7 + */ + public static StringType fromToken(IToken token) { + switch(token.getType()) { + case IToken.tSTRING: return NARROW; + case IToken.tLSTRING: return WIDE; + case IToken.tUTF16STRING: return UTF16; + case IToken.tUTF32STRING: return UTF32; + case IToken.tUSER_DEFINED_STRING_LITERAL: { + char[] image = token.getCharImage(); + switch (image[0]) { + case 'R': + case '"': return NARROW; + case 'L': return WIDE; + case 'u': + if (image.length > 3 && image[1] == '8') { + return NARROW; + } + return UTF16; + case 'U': return UTF32; + } + } + //$FALL-THROUGH$ + default: + throw new IllegalArgumentException(token.getType() + " is not a string token"); + } + } + + /** + * Returns the user-defined suffix of a user-define string literal + * @param token + * @return the suffix of the token, if it exists + * @since 5.7 + */ + public static String getUserDefinedLiteralSuffix(IToken token) { + if (token.getType() == IToken.tUSER_DEFINED_STRING_LITERAL) { + int offset = token.getImage().lastIndexOf('"'); + return token.getImage().substring(offset + 1); + } + return new String(); + } } diff --git a/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/ScannerExtensionConfiguration.java b/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/ScannerExtensionConfiguration.java index 502ec24b536..f4d06d80ed3 100644 --- a/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/ScannerExtensionConfiguration.java +++ b/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/ScannerExtensionConfiguration.java @@ -41,4 +41,8 @@ public class ScannerExtensionConfiguration extends AbstractScannerExtensionConfi return true; } + @Override + public boolean supportUserDefinedLiterals() { + return false; + } }