From 11d74c9995400d276f191cca99d0824228ab6a7b Mon Sep 17 00:00:00 2001 From: John Camelon Date: Fri, 2 Apr 2004 18:51:01 +0000 Subject: [PATCH] org.eclipse.cdt.core Updated SelectionSearch to work for functions, variables with initializers, etc. Fixed https://bugs.eclipse.org/bugs/show_bug.cgi?id=39705 Fixed https://bugs.eclipse.org/bugs/show_bug.cgi?id=44336 Fixed https://bugs.eclipse.org/bugs/show_bug.cgi?id=36770 Partial fix for https://bugs.eclipse.org/bugs/show_bug.cgi?id=51428 Fixed https://bugs.eclipse.org/bugs/show_bug.cgi?id=39694 org.eclipse.cdt.core.tests Added SelectionParseTest::testBaseCase_FunctionDeclaration(). Added SelectionParseTest::testBaseCase_FunctionDeclaration2(). Added SelectionParseTest::testBaseCase_VariableDeclaration(). Added SelectionParseTest::testBaseCase_Parameter(). Added QuickParseASTTests::testBug44336(). Added ScannerTestCase::testBug36770B(). Moved testBug39705() from ASTFailedTests to QuickParseASTTests. Moved testBug39694() from ASTFailedTests to QuickParseASTTests. --- core/org.eclipse.cdt.core.tests/ChangeLog | 10 ++ .../parser/failedTests/ASTFailedTests.java | 13 +- .../core/parser/tests/BaseScannerTest.java | 2 + .../core/parser/tests/QuickParseASTTests.java | 21 ++++ .../core/parser/tests/ScannerTestCase.java | 23 ++-- .../core/parser/tests/SelectionParseTest.java | 63 +++++++--- .../parser/ChangeLog-parser | 8 ++ .../eclipse/cdt/core/parser/ITokenDuple.java | 4 + .../parser/extension/IScannerExtension.java | 12 ++ .../cdt/internal/core/parser/Parser.java | 12 +- .../internal/core/parser/SelectionParser.java | 23 +++- .../parser/scanner/GCCScannerExtension.java | 82 +++++++++++-- .../internal/core/parser/scanner/Scanner.java | 116 +++++++++--------- .../core/parser/scanner/ScannerUtility.java | 24 ++-- .../core/parser/token/TokenDuple.java | 24 ++++ 15 files changed, 317 insertions(+), 120 deletions(-) diff --git a/core/org.eclipse.cdt.core.tests/ChangeLog b/core/org.eclipse.cdt.core.tests/ChangeLog index bd316582bcc..47cb165952c 100644 --- a/core/org.eclipse.cdt.core.tests/ChangeLog +++ b/core/org.eclipse.cdt.core.tests/ChangeLog @@ -1,3 +1,13 @@ +2004-04-02 John Camelon + Added SelectionParseTest::testBaseCase_FunctionDeclaration(). + Added SelectionParseTest::testBaseCase_FunctionDeclaration2(). + Added SelectionParseTest::testBaseCase_VariableDeclaration(). + Added SelectionParseTest::testBaseCase_Parameter(). + Added QuickParseASTTests::testBug44336(). + Added ScannerTestCase::testBug36770B(). + Moved testBug39705() from ASTFailedTests to QuickParseASTTests. + Moved testBug39694() from ASTFailedTests to QuickParseASTTests. + 2004-03-29 John Camelon Added ScannerTestCase::testBug56517(). diff --git a/core/org.eclipse.cdt.core.tests/failures/org/eclipse/cdt/core/parser/failedTests/ASTFailedTests.java b/core/org.eclipse.cdt.core.tests/failures/org/eclipse/cdt/core/parser/failedTests/ASTFailedTests.java index 1b65d508205..d1dca0671c7 100644 --- a/core/org.eclipse.cdt.core.tests/failures/org/eclipse/cdt/core/parser/failedTests/ASTFailedTests.java +++ b/core/org.eclipse.cdt.core.tests/failures/org/eclipse/cdt/core/parser/failedTests/ASTFailedTests.java @@ -116,13 +116,7 @@ public class ASTFailedTests extends BaseASTTest } assertCodeFailsParse(code.toString()); } - public void testBug39694() throws Exception - { - IASTVariable variable = (IASTVariable)parse("int ab$cd = 1;").getDeclarations().next(); - assertFalse( - "The expected error did not occur.", - variable.equals("ab$cd")); - } + public void testBug39695() throws Exception { assertCodeFailsParse("int a = __alignof__ (int);"); @@ -225,10 +219,7 @@ public class ASTFailedTests extends BaseASTTest { assertCodeFailsParse("__declspec(dllexport) int func1 (int a) {}"); } - public void testBug39705() throws Exception - { - assertCodeFailsParse("#ident \"@(#)filename.c 1.3 90/02/12\""); - } + public void testBug40422() throws Exception { diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScannerTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScannerTest.java index eafa711e922..05f170721dd 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScannerTest.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScannerTest.java @@ -184,6 +184,8 @@ public class BaseScannerTest extends TestCase { } } + + public void validateDefinition(String name, String value) { String definition= null; diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java index 6eab2ccf977..1a64ff65a55 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java @@ -2115,4 +2115,25 @@ public class QuickParseASTTests extends BaseASTTest assertFalse( i.hasNext() ); } + + public void testBug44336() throws Exception + { + Iterator i = parse( "class A {}; typedef typename A foo;" ).getDeclarations(); + IASTClassSpecifier classA = (IASTClassSpecifier) ((IASTAbstractTypeSpecifierDeclaration)i.next()).getTypeSpecifier(); + assertEquals( classA.getClassKind(), ASTClassKind.CLASS ); + assertEquals( classA.getName(), "A"); + IASTTypedefDeclaration typedefDeclaration = (IASTTypedefDeclaration) i.next(); + assertFalse( i.hasNext() ); + } + + public void testBug39705() throws Exception + { + parse("#ident \"@(#)filename.c 1.3 90/02/12\""); + } + + public void testBug39694() throws Exception + { + IASTVariable variable = (IASTVariable)parse("int ab$cd = 1;").getDeclarations().next(); + assertEquals( variable.getName(), "ab$cd"); + } } \ No newline at end of file diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java index d62f7e243b4..5c95fa5e1ff 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java @@ -1588,12 +1588,21 @@ public class ScannerTestCase extends BaseScannerTest writer.write( "#endif\n"); initializeScanner( writer.toString() ); validateEOF(); -// validateToken( IToken.t_char); -// validateToken( IToken.tSTAR); -// validateIdentifier( "x"); -// validateToken( IToken.tASSIGN ); -// validateString( "#boo"); -// validateToken(IToken.tSEMI); -// validateEOF(); + } + + public void testBug36770B() throws Exception + { + Writer writer = new StringWriter(); + writer.write( "#define A 0\n" ); + writer.write( "#if ( A == 1 )\n" ); + writer.write( "# define foo\n" ); + writer.write( "#else\n" ); + writer.write( "# define bar\n" ); + writer.write( "#endif\n" ); + initializeScanner( writer.toString(), ParserMode.QUICK_PARSE ); + validateEOF(); + validateDefinition( "A", 0 ); + validateDefinition( "bar", "" ); + } } diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/SelectionParseTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/SelectionParseTest.java index 28f8d6a10fd..6067c85b71e 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/SelectionParseTest.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/SelectionParseTest.java @@ -21,6 +21,7 @@ import org.eclipse.cdt.core.parser.ParserUtil; import org.eclipse.cdt.core.parser.ScannerInfo; import org.eclipse.cdt.core.parser.ast.IASTFunction; import org.eclipse.cdt.core.parser.ast.IASTNode; +import org.eclipse.cdt.core.parser.ast.IASTParameterDeclaration; import org.eclipse.cdt.core.parser.ast.IASTVariable; /** @@ -55,7 +56,7 @@ public class SelectionParseTest extends CompleteParseBaseTest { public void testBaseCase_VariableReference() throws Exception { - String code = "int x; x=3;"; + String code = "void f() { int x; x=3; }"; int offset1 = code.indexOf( "x=" ); int offset2 = code.indexOf( '='); IASTNode node = parse( code, offset1, offset2 ); @@ -81,25 +82,47 @@ public class SelectionParseTest extends CompleteParseBaseTest { assertNull( parse( code, offset1, offset2 )); } -// public void testBaseCase_FunctionDeclaration() throws Exception -// { -// String code = "int x(); x( );"; -// int offset1 = code.indexOf( "x()" ); -// int offset2 = code.indexOf( "()"); -// IASTNode node = parse( code, offset1, offset2 ); -// assertTrue( node instanceof IASTFunction ); -// assertEquals( ((IASTFunction)node).getName(), "x" ); -// } -// -// public void testBaseCase_VariableDeclaration() throws Exception -// { -// String code = "int x = 3;"; -// int offset1 = code.indexOf( "x" ); -// int offset2 = code.indexOf( " ="); -// IASTNode node = parse( code, offset1, offset2 ); -// assertTrue( node instanceof IASTVariable ); -// assertEquals( ((IASTVariable)node).getName(), "x" ); -// } + public void testBaseCase_FunctionDeclaration() throws Exception + { + String code = "int x(); x( );"; + int offset1 = code.indexOf( "x()" ); + int offset2 = code.indexOf( "()"); + IASTNode node = parse( code, offset1, offset2 ); + assertTrue( node instanceof IASTFunction ); + assertEquals( ((IASTFunction)node).getName(), "x" ); + } + + public void testBaseCase_FunctionDeclaration2() throws Exception + { + String code = "int printf( const char *, ... ); "; + int offset1 = code.indexOf( "printf" ); + int offset2 = code.indexOf( "( const"); + IASTNode node = parse( code, offset1, offset2 ); + assertTrue( node instanceof IASTFunction ); + assertEquals( ((IASTFunction)node).getName(), "printf" ); + } + + public void testBaseCase_VariableDeclaration() throws Exception + { + String code = "int x = 3;"; + int offset1 = code.indexOf( "x" ); + int offset2 = code.indexOf( " ="); + IASTNode node = parse( code, offset1, offset2 ); + assertNotNull( node ); + assertTrue( node instanceof IASTVariable ); + assertEquals( ((IASTVariable)node).getName(), "x" ); + } + + public void testBaseCase_Parameter() throws Exception + { + String code = "int main( int argc ) { int x = argc; }"; + int offset1 = code.indexOf( "argc;" ); + int offset2 = code.indexOf( ";" ); + IASTNode node = parse( code, offset1, offset2 ); + assertNotNull( node ); + assertTrue( node instanceof IASTParameterDeclaration ); + assertEquals( ((IASTParameterDeclaration)node).getName(), "argc" ); + } } diff --git a/core/org.eclipse.cdt.core/parser/ChangeLog-parser b/core/org.eclipse.cdt.core/parser/ChangeLog-parser index c4a27947c00..86b73eca41f 100644 --- a/core/org.eclipse.cdt.core/parser/ChangeLog-parser +++ b/core/org.eclipse.cdt.core/parser/ChangeLog-parser @@ -1,3 +1,11 @@ +2004-04-02 John Camelon + Updated SelectionSearch to work for functions, variables with initializers, etc. + Fixed https://bugs.eclipse.org/bugs/show_bug.cgi?id=39705 + Fixed https://bugs.eclipse.org/bugs/show_bug.cgi?id=44336 + Fixed https://bugs.eclipse.org/bugs/show_bug.cgi?id=36770 + Partial fix for https://bugs.eclipse.org/bugs/show_bug.cgi?id=51428 + Fixed https://bugs.eclipse.org/bugs/show_bug.cgi?id=39694 + 2004-03-30 John Camelon Partial fix for Bug 56614 - Content Assist] Global variables do not appear in a global completion list diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/ITokenDuple.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/ITokenDuple.java index 11074fe6cb9..d394e8a673e 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/ITokenDuple.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/ITokenDuple.java @@ -52,4 +52,8 @@ public interface ITokenDuple { public int getStartOffset(); public int getEndOffset(); public int getLineNumber(); + /** + * @return + */ + public abstract boolean syntaxOfName(); } \ No newline at end of file diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/extension/IScannerExtension.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/extension/IScannerExtension.java index 495607ffd85..61280ec8f9b 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/extension/IScannerExtension.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/extension/IScannerExtension.java @@ -24,4 +24,16 @@ public interface IScannerExtension extends Cloneable { public boolean canHandlePreprocessorDirective( String directive ); public void handlePreprocessorDirective( String directive, String restOfLine ); + + /** + * @return + */ + public boolean offersDifferentIdentifierCharacters(); + + /** + * @param c + * @return + */ + public boolean isValidIdentifierStartCharacter(int c); + public boolean isValidIdentifierCharacter( int c ); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java index acb06e57a79..f2f297eadc4 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java @@ -749,7 +749,6 @@ public abstract class Parser extends ExpressionParser implements IParser simpleDeclarationStrategyUnion(scope, ownerTemplate, overideKind ); } setCompletionValues(scope, kind, Key.DECLARATION ); - checkEndOfFile(); } /** @@ -1066,10 +1065,9 @@ public abstract class Parser extends ExpressionParser implements IParser { try { - astFactory - .createTypeSpecDeclaration( - sdw.getScope(), - sdw.getTypeSpecifier(), + astFactory.createTypeSpecDeclaration( + sdw.getScope(), + sdw.getTypeSpecifier(), ownerTemplate, sdw.getStartingOffset(), sdw.getStartingLine(), lastToken.getEndOffset(), lastToken.getLineNumber()) @@ -1551,9 +1549,11 @@ public abstract class Parser extends ExpressionParser implements IParser consume(IToken.t_template); last = templateId(sdw.getScope(), CompletionKind.SINGLE_NAME_REFERENCE ); } + if( sdw.getName() != null ) + first = sdw.getName().getFirstToken(); ITokenDuple duple = new TokenDuple(first, last); sdw.setTypeName(duple); - + flags.setEncounteredTypename(true); break; case IToken.tCOLONCOLON : consume(IToken.tCOLONCOLON); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/SelectionParser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/SelectionParser.java index dc1b3048219..960c3cdc36f 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/SelectionParser.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/SelectionParser.java @@ -20,6 +20,7 @@ import org.eclipse.cdt.core.parser.ParseError; import org.eclipse.cdt.core.parser.ParserLanguage; import org.eclipse.cdt.core.parser.ast.IASTCompletionNode; import org.eclipse.cdt.core.parser.ast.IASTNode; +import org.eclipse.cdt.core.parser.ast.IASTScope; import org.eclipse.cdt.internal.core.parser.token.OffsetDuple; import org.eclipse.cdt.internal.core.parser.token.TokenDuple; @@ -30,6 +31,9 @@ public class SelectionParser extends ContextualParser { private OffsetDuple offsetRange; private IToken firstTokenOfDuple = null, lastTokenOfDuple = null; + private IASTScope ourScope = null; + private IASTCompletionNode.CompletionKind ourKind = null; + private IASTNode ourContext = null; /* (non-Javadoc) * @see org.eclipse.cdt.internal.core.parser.Parser#handleNewToken(org.eclipse.cdt.core.parser.IToken) @@ -48,6 +52,16 @@ public class SelectionParser extends ContextualParser { log.traceLog( "Offset Ceiling Hit w/token \"" + value.getImage() + "\""); //$NON-NLS-1$ //$NON-NLS-2$ lastTokenOfDuple = value; } + if( scanner.isOnTopContext() && lastTokenOfDuple != null && lastTokenOfDuple.getEndOffset() >= offsetRange.getCeilingOffset() ) + { + if ( ourScope == null ) + ourScope = getCompletionScope(); + if( ourContext == null ) + ourContext = getCompletionContext(); + if( ourKind == null ) + ourKind = getCompletionKind(); + + } } } @@ -67,7 +81,6 @@ public class SelectionParser extends ContextualParser { * @see org.eclipse.cdt.core.parser.IParser#parse(int, int) */ public IASTNode parse(int startingOffset, int endingOffset) { -// scanner.setOffsetBoundary(endingOffset); offsetRange = new OffsetDuple( startingOffset, endingOffset ); translationUnit(); return reconcileTokenDuple(); @@ -84,7 +97,11 @@ public class SelectionParser extends ContextualParser { throw new ParseError( ParseError.ParseErrorKind.OFFSETDUPLE_UNREACHABLE ); ITokenDuple duple = new TokenDuple( firstTokenOfDuple, lastTokenOfDuple ); - return duple.lookup( astFactory, getCompletionScope() ); + + if( ! duple.syntaxOfName() ) + throw new ParseError( ParseError.ParseErrorKind.OFFSET_RANGE_NOT_NAME ); + + return duple.lookup( astFactory, ourScope ); } /* (non-Javadoc) @@ -99,8 +116,6 @@ public class SelectionParser extends ContextualParser { * @see org.eclipse.cdt.internal.core.parser.Parser#checkEndOfFile() */ protected void checkEndOfFile() throws EndOfFileException { - if( scanner.isOnTopContext() && lastTokenOfDuple != null && lastTokenOfDuple.getEndOffset() >= offsetRange.getCeilingOffset() ) - throw new EndOfFileException(); } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java index 8943a155090..6376044c8ca 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java @@ -11,11 +11,15 @@ package org.eclipse.cdt.internal.core.parser.scanner; import java.util.HashSet; +import java.util.Iterator; import java.util.Set; +import org.eclipse.cdt.core.parser.CodeReader; import org.eclipse.cdt.core.parser.IScanner; import org.eclipse.cdt.core.parser.ParserLanguage; +import org.eclipse.cdt.core.parser.ast.IASTInclusion; import org.eclipse.cdt.core.parser.extension.IScannerExtension; +import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionParseException; /** * @author jcamelon @@ -65,6 +69,7 @@ public class GCCScannerExtension implements IScannerExtension { directives = new HashSet(); directives.add( "#include_next" ); //$NON-NLS-1$ directives.add( "#warning"); //$NON-NLS-1$ + directives.add( "#ident"); //$NON-NLS-1$ } /* (non-Javadoc) @@ -82,17 +87,78 @@ public class GCCScannerExtension implements IScannerExtension { { scannerData.getLogService().traceLog( "GCCScannerExtension handling #include_next directive" ); //$NON-NLS-1$ // figure out the name of the current file and its path -// IScannerContext context = scannerData.getContextStack().getCurrentContext(); -// if( context.getKind() != IScannerContext.ContextKind.INCLUSION ) -// { -// //handle appropriate error -// } -// String fullInclusionPath = context.getFilename(); -// IASTInclusion inclusion = context.getExtension(); + IScannerContext context = scannerData.getContextStack().getCurrentContext(); + if( context.getKind() != IScannerContext.ContextKind.INCLUSION ) + return; + String fullInclusionPath = context.getFilename(); + IASTInclusion inclusion = context.getExtension(); + + Iterator iter = scannerData.getIncludePathNames().iterator(); + + while (iter.hasNext()) { + String path = (String)iter.next(); + String completePath = ScannerUtility.createReconciledPath(path, inclusion.getName() ); + if( completePath.equals( fullInclusionPath ) ) + break; + } + + ScannerUtility.InclusionDirective parsedDirective = null; + try { + parsedDirective = ScannerUtility.parseInclusionDirective( scannerData, this, restOfLine, scannerData.getContextStack().getCurrentContext().getOffset() ); + } catch (InclusionParseException e) { + return; + } + CodeReader duple = null; + // search through include paths + while (iter.hasNext()) { + String path = (String)iter.next(); + duple = ScannerUtility.createReaderDuple( path, parsedDirective.getFilename(), scannerData.getClientRequestor() ); + if( duple != null ) + break; + } + + if( duple != null ) + { + try + { + scannerData.getContextStack().updateContext(duple.getUnderlyingReader(), duple.getFilename(), ScannerContext.ContextKind.INCLUSION, inclusion, scannerData.getClientRequestor() ); + scannerData.getLogService().traceLog( "GCCScannerExtension handling #include_next directive successfully pushed on new include file" ); //$NON-NLS-1$ + } + catch (ContextException e1) + { + return; + } + } - // search through include paths } + else if( directive.equals( "#warning") || directive.equals("#ident")) + return; // good enough -- the rest of the line has been consumed + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.extension.IScannerExtension#offersDifferentIdentifierCharacters() + */ + public boolean offersDifferentIdentifierCharacters() { + return true; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.extension.IScannerExtension#isValidIdentifierStartCharacter(int) + */ + public boolean isValidIdentifierStartCharacter(int c) { + return Character.isLetter((char)c) || ( c == '_') || ( c == '$' ); + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.extension.IScannerExtension#isValidIdentifierCharacter(int) + */ + public boolean isValidIdentifierCharacter(int c) { + return ((c >= 'a') && (c <= 'z')) + || ((c >= 'A') && (c <= 'Z')) + || ((c >= '0') && (c <= '9')) + || (c == '_') || ( c== '$' ) || + Character.isUnicodeIdentifierPart( (char)c); } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java index e95ecb7c631..0502f11d548 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java @@ -1437,12 +1437,9 @@ public class Scanner implements IScanner { c = getChar(); // do the least expensive tests first! - while ( - ((c >= 'a') && (c <= 'z')) - || ((c >= 'A') && (c <= 'Z')) - || ((c >= '0') && (c <= '9')) - || (c == '_') || Character.isUnicodeIdentifierPart( (char)c) - ) { + while ( ( scannerExtension.offersDifferentIdentifierCharacters() && + scannerExtension.isValidIdentifierCharacter(c) ) || + isValidIdentifierCharacter(c) ) { buff.append((char) c); c = getChar(); if (c == '\\') { @@ -1491,6 +1488,17 @@ public class Scanner implements IScanner { return newToken(IToken.tIDENTIFIER, ident, scannerData.getContextStack().getCurrentContext()); } + /** + * @param c + * @return + */ + protected boolean isValidIdentifierCharacter(int c) { + return ((c >= 'a') && (c <= 'z')) + || ((c >= 'A') && (c <= 'Z')) + || ((c >= '0') && (c <= '9')) + || (c == '_') || Character.isUnicodeIdentifierPart( (char)c); + } + public IToken nextToken( boolean pasting ) throws ScannerException, EndOfFileException { if( ! initialContextInitialized ) @@ -1573,11 +1581,6 @@ public class Scanner implements IScanner { continue; } return token; - -// case '/': -// expandDefinition("??/", "\\", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$ -// c = getChar(insideString); -// break; default: // Not a trigraph ungetChar(c); @@ -1850,12 +1853,9 @@ public class Scanner implements IScanner { return token; default: - if ( - // JOHNC - Accept non-ASCII Input -// ((c >= 'a') && (c <= 'z')) -// || ((c >= 'A') && (c <= 'Z')) || (c == '_')) { - Character.isLetter((char)c) || ( c == '_') - ) + if ( ( scannerExtension.offersDifferentIdentifierCharacters() && + scannerExtension.isValidIdentifierStartCharacter(c) ) || + isValidIdentifierStartCharacter(c) ) { token = processKeywordOrLiteral(c, pasting); if (token == null) @@ -1885,6 +1885,14 @@ public class Scanner implements IScanner { /** + * @param c + * @return + */ + protected boolean isValidIdentifierStartCharacter(int c) { + return Character.isLetter((char)c) || ( c == '_'); + } + + /** * @param definition */ protected void handleCompletionOnDefinition(String definition) throws EndOfFileException { @@ -2292,47 +2300,43 @@ public class Scanner implements IScanner { protected boolean evaluateExpression(String expression, int beginningOffset ) throws ScannerException { - if( scannerData.getParserMode() == ParserMode.QUICK_PARSE ) - { - if( expression.trim().equals( "0" ) ) //$NON-NLS-1$ - return false; - - return true; - } - else - { - IExpressionParser parser = null; - StringBuffer expressionBuffer = new StringBuffer( expression ); - expressionBuffer.append( ';'); + IExpressionParser parser = null; + StringBuffer expressionBuffer = new StringBuffer( expression ); + expressionBuffer.append( ';'); - IScanner trial = new Scanner( - new StringReader(expressionBuffer.toString()), - EXPRESSION, - scannerData.getDefinitions(), - scannerData.getIncludePathNames(), - NULL_REQUESTOR, - ParserMode.QUICK_PARSE, - scannerData.getLanguage(), - NULL_LOG_SERVICE, - scannerExtension ); - - parser = InternalParserUtil.createExpressionParser(trial, scannerData.getLanguage(), NULL_LOG_SERVICE); - try { - IASTExpression exp = parser.expression(null); - if( exp.evaluateExpression() == 0 ) - return false; - return true; - } catch( BacktrackException backtrack ) - { - handleProblem( IProblem.PREPROCESSOR_CONDITIONAL_EVAL_ERROR, expression, beginningOffset, false, true ); - } - catch (ASTExpressionEvaluationException e) { - handleProblem( IProblem.PREPROCESSOR_CONDITIONAL_EVAL_ERROR, expression, beginningOffset, false, true ); - } catch (EndOfFileException e) { - handleProblem( IProblem.PREPROCESSOR_CONDITIONAL_EVAL_ERROR, expression, beginningOffset, false, true ); - } - return true; + IScanner trial = new Scanner( + new StringReader(expressionBuffer.toString()), + EXPRESSION, + scannerData.getDefinitions(), + scannerData.getIncludePathNames(), + NULL_REQUESTOR, + ParserMode.QUICK_PARSE, + scannerData.getLanguage(), + NULL_LOG_SERVICE, + scannerExtension ); + + parser = InternalParserUtil.createExpressionParser(trial, scannerData.getLanguage(), NULL_LOG_SERVICE); + try { + IASTExpression exp = parser.expression(null); + if( exp.evaluateExpression() == 0 ) + return false; + return true; + } catch( BacktrackException backtrack ) + { + if( scannerData.getParserMode() == ParserMode.QUICK_PARSE ) + return false; + handleProblem( IProblem.PREPROCESSOR_CONDITIONAL_EVAL_ERROR, expression, beginningOffset, false, true ); } + catch (ASTExpressionEvaluationException e) { + if( scannerData.getParserMode() == ParserMode.QUICK_PARSE ) + return false; + handleProblem( IProblem.PREPROCESSOR_CONDITIONAL_EVAL_ERROR, expression, beginningOffset, false, true ); + } catch (EndOfFileException e) { + if( scannerData.getParserMode() == ParserMode.QUICK_PARSE ) + return false; + handleProblem( IProblem.PREPROCESSOR_CONDITIONAL_EVAL_ERROR, expression, beginningOffset, false, true ); + } + return true; } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java index b7109e3fbd7..d1d3494fcf6 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java @@ -62,22 +62,30 @@ public class ScannerUtility { return buffer.toString(); } + static CodeReader createReaderDuple( String path, String fileName, ISourceElementRequestor requestor ) { - File pathFile = new File(path); - //TODO assert pathFile.isDirectory(); - StringBuffer newPathBuffer = new StringBuffer( pathFile.getPath() ); - newPathBuffer.append( File.separatorChar ); - newPathBuffer.append( fileName ); - //remove ".." and "." segments - String finalPath = reconcilePath( newPathBuffer.toString() ); + String finalPath = createReconciledPath(path, fileName); Reader r = requestor.createReader( finalPath ); if( r != null ) return new CodeReader( finalPath, r ); return null; - } + /** + * @param path + * @param fileName + * @return + */ + static String createReconciledPath(String path, String fileName) { + //TODO assert pathFile.isDirectory(); + StringBuffer newPathBuffer = new StringBuffer( new File(path).getPath() ); + newPathBuffer.append( File.separatorChar ); + newPathBuffer.append( fileName ); + //remove ".." and "." segments + return reconcilePath( newPathBuffer.toString() ); + } + static class InclusionDirective { public InclusionDirective( String fileName, boolean useIncludePaths, int startOffset, int endOffset ) diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenDuple.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenDuple.java index a5e0c7d4fa6..9d0d8618516 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenDuple.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenDuple.java @@ -402,4 +402,28 @@ public class TokenDuple implements ITokenDuple { public List[] getTemplateIdArgLists() { return argLists; } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.ITokenDuple#syntaxOfName() + */ + public boolean syntaxOfName() { + Iterator iter = iterator(); + if( ! iter.hasNext() ) return false; // empty is not good + while( iter.hasNext() ) + { + IToken token = (IToken) iter.next(); + if( token.isOperator() ) continue; + switch( token.getType() ) + { + case IToken.tCOMPL: + case IToken.tIDENTIFIER: + case IToken.tCOLONCOLON: + case IToken.t_operator: + continue; + default: + return false; + } + } + return true; + } }