From e92ee9c4db78dd013ccbcfa9bfc5ff45afec5212 Mon Sep 17 00:00:00 2001 From: Doug Schaefer Date: Thu, 17 Jun 2004 15:25:34 +0000 Subject: [PATCH] Added undef support, including remove method for CharArrayMaps. --- .../tests/scanner2/Scanner2SpeedTest.java | 2 +- .../parser/tests/scanner2/Scanner2Test.java | 19 ++ .../core/parser/scanner2/CharArrayMap.java | 38 +++ .../parser/scanner2/CharArrayObjectMap.java | 25 ++ .../core/parser/scanner2/Scanner2.java | 242 ++++++++++++++++-- 5 files changed, 305 insertions(+), 21 deletions(-) diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2SpeedTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2SpeedTest.java index 550f6e0f106..17fc4b20fbc 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2SpeedTest.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2SpeedTest.java @@ -54,7 +54,7 @@ public class Scanner2SpeedTest extends SpeedTest2 { } private void runTest(int n) throws Exception { - String code = + String code = "#include \n" + "#include \n" + "#include \n"; diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java index 26aa54d7e89..0877a7a8ee8 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java @@ -1640,4 +1640,23 @@ public class Scanner2Test extends BaseScanner2Test validateToken( IToken.tRBRACE); validateEOF(); } + + public void test__attribute__() throws Exception { + initializeScanner( + "#define __cdecl __attribute__((cdecl))\n" + + "__cdecl;"); + validateToken(IToken.tSEMI); + validateEOF(); + } + + public void testUndef() throws Exception { + initializeScanner( + "#define A 5\n" + + "#define B 10\n" + + "#undef A\n" + + "A B"); + validateIdentifier("A"); + validateInteger("10"); + validateEOF(); + } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayMap.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayMap.java index 4ae356869c3..b7b697dd9be 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayMap.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayMap.java @@ -38,6 +38,10 @@ public abstract class CharArrayMap { return CharArrayUtils.hash(buffer, start, len) & (keyTable.length - 1); } + private int hash(char[] buffer) { + return hash(buffer, 0, buffer.length); + } + private void insert(int i) { insert(i, hash(keyTable[i], 0, keyTable[i].length)); } @@ -124,6 +128,40 @@ public abstract class CharArrayMap { return -1; } + protected void removeEntry(int i) { + // Remove the hash entry + int hash = hash(keyTable[i]); + if (hashTable[hash] == i + 1) + hashTable[hash] = nextTable[i]; + else { + // find entry pointing to me + int j = hashTable[hash] - 1; + while (nextTable[j] != 0 && nextTable[j] != i + 1) + j = nextTable[j] - 1; + nextTable[j] = nextTable[i]; + } + + if (i < currEntry) { + // shift everything over + System.arraycopy(keyTable, i + 1, keyTable, i, currEntry - i); + System.arraycopy(nextTable, i + 1, nextTable, i, currEntry - i); + + // adjust hash and next entries for things that moved + for (int j = 0; j < hashTable.length; ++j) + if (hashTable[j] > i) + --hashTable[j]; + + for (int j = 0; j < nextTable.length; ++j) + if (nextTable[j] > i) + --nextTable[j]; + } + + // last entry is now free + keyTable[currEntry] = null; + nextTable[currEntry] = 0; + --currEntry; + } + public void dumpNexts() { for (int i = 0; i < nextTable.length; ++i) { if (nextTable[i] == 0) diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayObjectMap.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayObjectMap.java index 4a0b339ae5e..b3741ae6bba 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayObjectMap.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayObjectMap.java @@ -52,4 +52,29 @@ public class CharArrayObjectMap extends CharArrayMap { return get(key, 0, key.length); } + public Object remove(char[] key, int start, int length) { + int i = lookup(key, start, length); + if (i < 0) + return null; + + Object value = valueTable[i]; + + removeEntry(i); + + return value; + } + + public Object remove(char[] key) { + return remove(key, 0, key.length); + } + + protected void removeEntry(int i) { + // Remove the entry from the valueTable, shifting everything over if necessary + if (i < currEntry) + System.arraycopy(valueTable, i + 1, valueTable, i, currEntry - i); + valueTable[currEntry] = null; + + // Make sure you remove the value before calling super where currEntry will change + super.removeEntry(i); + } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java index c72aab49d02..b87fbd4597f 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java @@ -52,7 +52,7 @@ public class Scanner2 implements IScanner, IScannerData { protected IParserLogService log; private IScannerExtension scannerExtension; private List workingCopies; - private CharArrayObjectMap definitions; + private CharArrayObjectMap definitions = new CharArrayObjectMap(64); private String[] includePaths; int count; @@ -93,10 +93,11 @@ public class Scanner2 implements IScanner, IScannerData { pushContext(reader.buffer, reader); + setupBuiltInMacros(); + if (info.getDefinedSymbols() != null) { Map symbols = info.getDefinedSymbols(); String[] keys = (String[])symbols.keySet().toArray(emptyStringArray); - definitions = new CharArrayObjectMap(symbols.size()); for (int i = 0; i < keys.length; ++i) { String symbolName = (String)keys[i]; Object value = symbols.get(symbolName); @@ -107,8 +108,6 @@ public class Scanner2 implements IScanner, IScannerData { } else if( value instanceof IMacroDescriptor ) addDefinition( symbolName, (IMacroDescriptor)value); } - } else { - definitions = new CharArrayObjectMap(4); } includePaths = info.getIncludePaths(); @@ -952,6 +951,9 @@ public class Scanner2 implements IScanner, IScannerData { case ppDefine: handlePPDefine(); return; + case ppUndef: + handlePPUndef(); + return; case ppIfdef: handlePPIfdef(true); return; @@ -1161,6 +1163,41 @@ public class Scanner2 implements IScanner, IScannerData { : new FunctionStyleMacro(name, text, arglist)); } + private void handlePPUndef() { + char[] buffer = bufferStack[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + + skipOverWhiteSpace(); + + // get the Identifier + int idstart = ++bufferPos[bufferStackPos]; + if (idstart >= limit) + return; + + char c = buffer[idstart]; + if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_')) { + skipToNewLine(); + return; + } + + int idlen = 1; + while (++bufferPos[bufferStackPos] < limit) { + c = buffer[bufferPos[bufferStackPos]]; + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || c == '_' || (c >= '0' && c <= '9')) { + ++idlen; + continue; + } else { + break; + } + } + --bufferPos[bufferStackPos]; + + skipToNewLine(); + + definitions.remove(buffer, idstart, idlen); + } + private void handlePPIfdef(boolean positive) { char[] buffer = bufferStack[bufferStackPos]; int limit = bufferLimit[bufferStackPos]; @@ -1436,11 +1473,27 @@ public class Scanner2 implements IScanner, IScannerData { return; boolean escaped = false; - while (++bufferPos[bufferStackPos] < limit) + boolean inComment = false; + while (++bufferPos[bufferStackPos] < limit) { switch (buffer[bufferPos[bufferStackPos]]) { + case '/': + pos = bufferPos[bufferStackPos]; + if (pos + 1 < limit && buffer[pos + 1] == '*') { + ++bufferPos[bufferStackPos]; + while (++bufferPos[bufferStackPos] < limit) { + pos = bufferPos[bufferStackPos]; + if (buffer[pos] == '*' + && pos + 1 < limit + && buffer[pos + 1] == '/') { + ++bufferPos[bufferStackPos]; + break; + } + } + } + break; case '\\': escaped = !escaped; - break; + continue; case '\n': if (escaped) { escaped = false; @@ -1448,9 +1501,9 @@ public class Scanner2 implements IScanner, IScannerData { } else { return; } - default: - escaped = false; } + escaped = false; + } } private void handleFunctionStyleMacro(FunctionStyleMacro macro) { @@ -1464,7 +1517,6 @@ public class Scanner2 implements IScanner, IScannerData { FunctionStyleMacro.Expansion exp = macro.new Expansion(); char[][] arglist = macro.arglist; int currarg = -1; - int parens = 0; while (bufferPos[bufferStackPos] < limit) { if (++currarg >= arglist.length || arglist[currarg] == null) @@ -1476,22 +1528,14 @@ public class Scanner2 implements IScanner, IScannerData { int pos = ++bufferPos[bufferStackPos]; char c = buffer[pos]; if (c == ')') { - if (parens == 0) - // end of macro - break; - else { - --parens; - continue; - } + // end of macro + break; } else if (c == ',') { // empty arg exp.definitions.put(arglist[currarg], emptyCharArray); continue; - } else if (c == '(') { - ++parens; - continue; } - + // peel off the arg --bufferPos[bufferStackPos]; int argend = bufferPos[bufferStackPos]; @@ -1534,6 +1578,164 @@ public class Scanner2 implements IScanner, IScannerData { pushContext(expText, exp); } + // gcc built-ins + private static final ObjectStyleMacro __cplusplus + = new ObjectStyleMacro("__cplusplus".toCharArray(), "1".toCharArray()); + private static final ObjectStyleMacro __STDC_HOSTED__ + = new ObjectStyleMacro("__STDC_HOSTED__".toCharArray(), "0".toCharArray()); + private static final ObjectStyleMacro __STDC_VERSION__ + = new ObjectStyleMacro("__STDC_VERSION__".toCharArray(), "199001L".toCharArray()); + private static final FunctionStyleMacro __attribute__ + = new FunctionStyleMacro( + "__attribute__".toCharArray(), + emptyCharArray, + new char[][] { "arg".toCharArray() }); + + // standard built-ins + private static final ObjectStyleMacro __STDC__ + = new ObjectStyleMacro("__STDC__".toCharArray(), "1".toCharArray()); + + protected void setupBuiltInMacros() { + + // gcc extensions + if( language == ParserLanguage.CPP ) + definitions.put(__cplusplus.name, __cplusplus); + definitions.put(__STDC_HOSTED__.name, __STDC_HOSTED__); + definitions.put(__STDC_VERSION__.name, __STDC_VERSION__); + definitions.put(__attribute__.name, __attribute__); + + /* + + // add these to private table + if( scannerData.getScanner().getDefinition( __ATTRIBUTE__) == null ) + scannerData.getPrivateDefinitions().put( __ATTRIBUTE__, ATTRIBUTE_MACRO); + + if( scannerData.getScanner().getDefinition( __DECLSPEC) == null ) + scannerData.getPrivateDefinitions().put( __DECLSPEC, DECLSPEC_MACRO ); + + if( scannerData.getScanner().getDefinition( __EXTENSION__ ) == null ) + scannerData.getPrivateDefinitions().put( __EXTENSION__, EXTENSION_MACRO); + + if( scannerData.getScanner().getDefinition( __CONST__ ) == null ) + scannerData.getPrivateDefinitions().put( __CONST__, __CONST__MACRO); + if( scannerData.getScanner().getDefinition( __CONST ) == null ) + scannerData.getPrivateDefinitions().put( __CONST, __CONST_MACRO); + if( scannerData.getScanner().getDefinition( __INLINE__ ) == null ) + scannerData.getPrivateDefinitions().put( __INLINE__, __INLINE__MACRO); + if( scannerData.getScanner().getDefinition( __SIGNED__ ) == null ) + scannerData.getPrivateDefinitions().put( __SIGNED__, __SIGNED__MACRO); + if( scannerData.getScanner().getDefinition( __VOLATILE__ ) == null ) + scannerData.getPrivateDefinitions().put( __VOLATILE__, __VOLATILE__MACRO); + ObjectMacroDescriptor __RESTRICT_MACRO = new ObjectMacroDescriptor( __RESTRICT, Keywords.RESTRICT ); + if( scannerData.getScanner().getDefinition( __RESTRICT ) == null ) + scannerData.getPrivateDefinitions().put( __RESTRICT, __RESTRICT_MACRO); + if( scannerData.getScanner().getDefinition( __RESTRICT__ ) == null ) + scannerData.getPrivateDefinitions().put( __RESTRICT__, __RESTRICT__MACRO); + if( scannerData.getScanner().getDefinition( __TYPEOF__ ) == null ) + scannerData.getPrivateDefinitions().put( __TYPEOF__, __TYPEOF__MACRO); + if( scannerData.getLanguage() == ParserLanguage.CPP ) + if( scannerData.getScanner().getDefinition( __ASM__ ) == null ) + scannerData.getPrivateDefinitions().put( __ASM__, __ASM__MACRO); + */ + + // standard extensions + definitions.put(__STDC__.name, __STDC__); + + /* + if( getDefinition(__STDC__) == null ) + addDefinition( __STDC__, STDC_MACRO ); + + if( language == ParserLanguage.C ) + { + if( getDefinition(__STDC_HOSTED__) == null ) + addDefinition( __STDC_HOSTED__, STDC_HOSTED_MACRO); + if( getDefinition( __STDC_VERSION__) == null ) + addDefinition( __STDC_VERSION__, STDC_VERSION_MACRO); + } + else + if( getDefinition( __CPLUSPLUS ) == null ) + addDefinition( __CPLUSPLUS, CPLUSPLUS_MACRO); //$NON-NLS-1$ + + if( getDefinition(__FILE__) == null ) + addDefinition( __FILE__, + new DynamicMacroDescriptor( __FILE__, new DynamicMacroEvaluator() { + public String execute() { + return contextStack.getMostRelevantFileContext().getContextName(); + } + } ) ); + + if( getDefinition( __LINE__) == null ) + addDefinition( __LINE__, + new DynamicMacroDescriptor( __LINE__, new DynamicMacroEvaluator() { + public String execute() { + return new Integer( contextStack.getCurrentLineNumber() ).toString(); + } + } ) ); + + + if( getDefinition( __DATE__ ) == null ) + addDefinition( __DATE__, + new DynamicMacroDescriptor( __DATE__, new DynamicMacroEvaluator() { + + public String getMonth() + { + if( Calendar.MONTH == Calendar.JANUARY ) return "Jan" ; //$NON-NLS-1$ + if( Calendar.MONTH == Calendar.FEBRUARY) return "Feb"; //$NON-NLS-1$ + if( Calendar.MONTH == Calendar.MARCH) return "Mar"; //$NON-NLS-1$ + if( Calendar.MONTH == Calendar.APRIL) return "Apr"; //$NON-NLS-1$ + if( Calendar.MONTH == Calendar.MAY) return "May"; //$NON-NLS-1$ + if( Calendar.MONTH == Calendar.JUNE) return "Jun"; //$NON-NLS-1$ + if( Calendar.MONTH == Calendar.JULY) return "Jul"; //$NON-NLS-1$ + if( Calendar.MONTH == Calendar.AUGUST) return "Aug"; //$NON-NLS-1$ + if( Calendar.MONTH == Calendar.SEPTEMBER) return "Sep"; //$NON-NLS-1$ + if( Calendar.MONTH == Calendar.OCTOBER) return "Oct"; //$NON-NLS-1$ + if( Calendar.MONTH == Calendar.NOVEMBER) return "Nov"; //$NON-NLS-1$ + if( Calendar.MONTH == Calendar.DECEMBER) return "Dec"; //$NON-NLS-1$ + return ""; //$NON-NLS-1$ + } + + public String execute() { + StringBuffer result = new StringBuffer(); + result.append( getMonth() ); + result.append(" "); //$NON-NLS-1$ + if( Calendar.DAY_OF_MONTH < 10 ) + result.append(" "); //$NON-NLS-1$ + result.append(Calendar.DAY_OF_MONTH); + result.append(" "); //$NON-NLS-1$ + result.append( Calendar.YEAR ); + return result.toString(); + } + } ) ); + + if( getDefinition( __TIME__) == null ) + addDefinition( __TIME__, + new DynamicMacroDescriptor( __TIME__, new DynamicMacroEvaluator() { + + + public String execute() { + StringBuffer result = new StringBuffer(); + if( Calendar.AM_PM == Calendar.PM ) + result.append( Calendar.HOUR + 12 ); + else + { + if( Calendar.HOUR < 10 ) + result.append( '0'); + result.append(Calendar.HOUR); + } + result.append(':'); + if( Calendar.MINUTE < 10 ) + result.append( '0'); + result.append(Calendar.MINUTE); + result.append(':'); + if( Calendar.SECOND < 10 ) + result.append( '0'); + result.append(Calendar.SECOND); + return result.toString(); + } + } ) ); + + */ + } /* (non-Javadoc) * @see org.eclipse.cdt.core.parser.IScanner#nextToken(boolean)