From ce1fa180df02eef40b842be7e09484f31789c143 Mon Sep 17 00:00:00 2001 From: Markus Schorn Date: Mon, 7 Jan 2013 14:53:00 +0100 Subject: [PATCH] Bug 397127: Limit raw-string detection to c++ parser. --- .../cdt/core/parser/tests/ast2/AST2Tests.java | 5 +++ .../core/parser/tests/scanner/LexerTests.java | 30 +++++++++--------- ...AbstractScannerExtensionConfiguration.java | 12 +++++-- .../IScannerExtensionConfiguration.java | 6 ++++ .../cpp/GPPScannerExtensionConfiguration.java | 8 +++++ .../core/parser/scanner/CPreprocessor.java | 1 + .../internal/core/parser/scanner/Lexer.java | 31 +++++++++++-------- 7 files changed, 64 insertions(+), 29 deletions(-) diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2Tests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2Tests.java index 23061d5fdf7..0a8e2de6c42 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2Tests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2Tests.java @@ -7420,4 +7420,9 @@ public class AST2Tests extends AST2BaseTest { public void testGCCDecltype_397227() throws Exception { parseAndCheckBindings(getAboveComment(), CPP, true); } + + // #define macro(R) #R"" + public void testNoRawStringInPlainC_397127() throws Exception { + parseAndCheckBindings(getAboveComment(), ParserLanguage.C, true); + } } diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java index aba5c29b8ff..23bdc2bd11a 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java @@ -27,10 +27,12 @@ public class LexerTests extends BaseTestCase { private static final LexerOptions NO_DOLLAR = new LexerOptions(); private static final LexerOptions NO_MINMAX = new LexerOptions(); private static final LexerOptions SLASH_PERCENT = new LexerOptions(); + private static final LexerOptions CPP_OPTIONS = new LexerOptions(); static { NO_DOLLAR.fSupportDollarInIdentifiers= false; NO_MINMAX.fSupportMinAndMax= false; SLASH_PERCENT.fSupportSlashPercentComments= true; + CPP_OPTIONS.fSupportRawStringLiterals= true; } static String TRIGRAPH_REPLACES_CHARS= "#^[]|{}~\\"; @@ -41,7 +43,7 @@ public class LexerTests extends BaseTestCase { } private Lexer fLexer; - private TestLexerLog fLog= new TestLexerLog(); + private final TestLexerLog fLog= new TestLexerLog(); private int fLastEndOffset; public LexerTests() { @@ -576,51 +578,51 @@ public class LexerTests extends BaseTestCase { public void testRawStringLiteral() throws Exception { String lit= "abc0123\\\"'.:; \\\\ \n\"("; - init("R\"(" + lit + ")\""); + init("R\"(" + lit + ")\"", CPP_OPTIONS); rstr("", lit); eof(); - init("LR\"(" + lit + ")\""); + init("LR\"(" + lit + ")\"", CPP_OPTIONS); wrstr("", lit); eof(); - init("u8R\"(" + lit + ")\""); + init("u8R\"(" + lit + ")\"", CPP_OPTIONS); utf8rstr("", lit); eof(); - init("uR\"(" + lit + ")\""); + init("uR\"(" + lit + ")\"", CPP_OPTIONS); utf16rstr("", lit); eof(); - init("UR\"(" + lit + ")\""); + init("UR\"(" + lit + ")\"", CPP_OPTIONS); utf32rstr("", lit); eof(); - init("R\"ut"); + init("R\"ut", CPP_OPTIONS); problem(IProblem.SCANNER_UNBOUNDED_STRING, "R\"ut"); token(IToken.tSTRING, "R\"ut"); eof(); - init("LR\"(ut"); + init("LR\"(ut", CPP_OPTIONS); problem(IProblem.SCANNER_UNBOUNDED_STRING, "LR\"(ut"); token(IToken.tLSTRING, "LR\"(ut"); eof(); - init("uR\"p()"); + init("uR\"p()", CPP_OPTIONS); problem(IProblem.SCANNER_UNBOUNDED_STRING, "uR\"p()"); token(IToken.tUTF16STRING, "uR\"p()"); eof(); - init("UR\"(ut"); + init("UR\"(ut", CPP_OPTIONS); problem(IProblem.SCANNER_UNBOUNDED_STRING, "UR\"(ut"); token(IToken.tUTF32STRING, "UR\"(ut"); eof(); - init("R\"+=(Text)=+\"Text)+=\""); + init("R\"+=(Text)=+\"Text)+=\"", CPP_OPTIONS); rstr("+=", "Text)=+\"Text"); eof(); - init("UR uR LR u8R U8R\"\""); + init("UR uR LR u8R U8R\"\"", CPP_OPTIONS); id("UR"); ws(); id("uR"); ws(); id("LR"); ws(); @@ -630,7 +632,7 @@ public class LexerTests extends BaseTestCase { } public void testRawStringLiteralInInactiveCode() throws Exception { - init("start\n" + "inactive: Rbla\n" + "#end"); + init("start\n" + "inactive: Rbla\n" + "#end", CPP_OPTIONS); id("start"); nextDirective(); token(IToken.tPOUND); @@ -638,7 +640,7 @@ public class LexerTests extends BaseTestCase { eof(); // raw string containing a directive - init("start\n" + "inactive: uR\"(\n#endif\n)\"\n" + "#end"); + init("start\n" + "inactive: uR\"(\n#endif\n)\"\n" + "#end", CPP_OPTIONS); id("start"); nextDirective(); token(IToken.tPOUND); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java index e9ec9210087..7cd17177aeb 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/AbstractScannerExtensionConfiguration.java @@ -29,8 +29,8 @@ public abstract class AbstractScannerExtensionConfiguration implements IScannerE private CharArrayIntMap fAddPreprocessorKeywords; protected static class MacroDefinition implements IMacro { - private char[] fSignature; - private char[] fExpansion; + private final char[] fSignature; + private final char[] fExpansion; MacroDefinition(char[] signature, char[] expansion) { fSignature= signature; @@ -103,6 +103,14 @@ public abstract class AbstractScannerExtensionConfiguration implements IScannerE return false; } + /** + * @since 5.5 + */ + @Override + public boolean supportRawStringLiterals() { + return false; + } + @Override public CharArrayIntMap getAdditionalPreprocessorKeywords() { return fAddPreprocessorKeywords; diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java index c4a2aa58afb..839dbbbd123 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/IScannerExtensionConfiguration.java @@ -109,4 +109,10 @@ public interface IScannerExtensionConfiguration { * @see "http://publib.boulder.ibm.com/infocenter/comphelp/v101v121/index.jsp?topic=/com.ibm.xlcpp101.aix.doc/language_ref/unicode_standard.html" */ public boolean supportUTFLiterals(); + + /** + * Support for C++ raw string literals. + * @since 5.5 + */ + public boolean supportRawStringLiterals(); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java index ffd4111d06a..476a245dc5e 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/parser/cpp/GPPScannerExtensionConfiguration.java @@ -113,4 +113,12 @@ public class GPPScannerExtensionConfiguration extends GNUScannerExtensionConfigu public boolean supportMinAndMaxOperators() { return true; } + + /** + * @since 5.5 + */ + @Override + public boolean supportRawStringLiterals() { + return true; + } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java index ad0bcd7b0aa..66382144eba 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java @@ -290,6 +290,7 @@ public class CPreprocessor implements ILexerLog, IScanner, IAdaptable { fLexOptions.fSupportMinAndMax = configuration.supportMinAndMaxOperators(); fLexOptions.fSupportSlashPercentComments= configuration.supportSlashPercentComments(); fLexOptions.fSupportUTFLiterals = configuration.supportUTFLiterals(); + fLexOptions.fSupportRawStringLiterals = configuration.supportRawStringLiterals(); fLocationMap= new LocationMap(fLexOptions); fKeywords= new CharArrayIntMap(40, -1); fPPKeywords= new CharArrayIntMap(40, -1); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java index 6b925714106..439e2a910fb 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java @@ -54,6 +54,7 @@ final public class Lexer implements ITokenSequence { public boolean fCreateImageLocations= true; public boolean fSupportSlashPercentComments= false; public boolean fSupportUTFLiterals= true; + public boolean fSupportRawStringLiterals= false; @Override public Object clone() { @@ -73,7 +74,7 @@ final public class Lexer implements ITokenSequence { // the input to the lexer private final AbstractCharArray fInput; - private int fStart; + private final int fStart; private int fLimit; // after phase 3 (newline, trigraph, line-splice) @@ -267,12 +268,14 @@ final public class Lexer implements ITokenSequence { case 'L': switch(d) { case 'R': - markPhase3(); - if (nextCharPhase3() == '"') { - nextCharPhase3(); - return rawStringLiteral(start, 3, IToken.tLSTRING); + if (fOptions.fSupportRawStringLiterals) { + markPhase3(); + if (nextCharPhase3() == '"') { + nextCharPhase3(); + return rawStringLiteral(start, 3, IToken.tLSTRING); + } + restorePhase3(); } - restorePhase3(); break; case '"': nextCharPhase3(); @@ -288,12 +291,14 @@ final public class Lexer implements ITokenSequence { if (fOptions.fSupportUTFLiterals) { switch(d) { case 'R': - markPhase3(); - if (nextCharPhase3() == '"') { - nextCharPhase3(); - return rawStringLiteral(start, 3, c == 'u' ? IToken.tUTF16STRING : IToken.tUTF32STRING); + if (fOptions.fSupportRawStringLiterals) { + markPhase3(); + if (nextCharPhase3() == '"') { + nextCharPhase3(); + return rawStringLiteral(start, 3, c == 'u' ? IToken.tUTF16STRING : IToken.tUTF32STRING); + } + restorePhase3(); } - restorePhase3(); break; case '"': nextCharPhase3(); @@ -306,7 +311,7 @@ final public class Lexer implements ITokenSequence { markPhase3(); switch (nextCharPhase3()) { case 'R': - if (nextCharPhase3() == '"') { + if (fOptions.fSupportRawStringLiterals && nextCharPhase3() == '"') { nextCharPhase3(); return rawStringLiteral(start, 4, IToken.tSTRING); } @@ -323,7 +328,7 @@ final public class Lexer implements ITokenSequence { return identifier(start, 1); case 'R': - if (d == '"') { + if (fOptions.fSupportRawStringLiterals && d == '"') { nextCharPhase3(); return rawStringLiteral(start, 2, IToken.tSTRING); }