From 551a9b7baef5ba4106ef7c77e460bd7a38bd0447 Mon Sep 17 00:00:00 2001 From: John Camelon Date: Thu, 15 Jul 2004 13:58:08 +0000 Subject: [PATCH] Further progress w/Scanner2. --- .../parser/tests/scanner2/Scanner2Test.java | 2 +- .../core/parser/scanner2/CharArrayUtils.java | 125 ++++++++++++++++++ .../core/parser/scanner2/Scanner2.java | 88 +++++++++++- 3 files changed, 208 insertions(+), 7 deletions(-) diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java index b9fabfc627a..e03882af802 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java @@ -278,7 +278,7 @@ public class Scanner2Test extends BaseScanner2Test initializeScanner("#define MULTICOMMENT X /* comment1 */ + Y /* comment 2 */"); //$NON-NLS-1$ validateEOF(); - validateDefinition("MULTICOMMENT", "X /* comment1 */ + Y"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("MULTICOMMENT", "X + Y"); //$NON-NLS-1$ //$NON-NLS-2$ initializeScanner("#define SIMPLE_STRING This is a simple string.\n"); //$NON-NLS-1$ validateEOF(); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayUtils.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayUtils.java index 2292f6206bb..356abd50605 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayUtils.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayUtils.java @@ -71,4 +71,129 @@ public class CharArrayUtils { System.arraycopy(str, start, copy, 0, length); return copy; } + + public static final char[] concat(char[] first, char[] second) { + if (first == null) + return second; + if (second == null) + return first; + + int length1 = first.length; + int length2 = second.length; + char[] result = new char[length1 + length2]; + System.arraycopy(first, 0, result, 0, length1); + System.arraycopy(second, 0, result, length1, length2); + return result; + } + + public static final char[] replace( + char[] array, + char[] toBeReplaced, + char[] replacementChars) { + + int max = array.length; + int replacedLength = toBeReplaced.length; + int replacementLength = replacementChars.length; + + int[] starts = new int[5]; + int occurrenceCount = 0; + + if (!equals(toBeReplaced, replacementChars)) { + + next : for (int i = 0; i < max; i++) { + int j = 0; + while (j < replacedLength) { + if (i + j == max) + continue next; + if (array[i + j] != toBeReplaced[j++]) + continue next; + } + if (occurrenceCount == starts.length) { + System.arraycopy( + starts, + 0, + starts = new int[occurrenceCount * 2], + 0, + occurrenceCount); + } + starts[occurrenceCount++] = i; + } + } + if (occurrenceCount == 0) + return array; + char[] result = + new char[max + + occurrenceCount * (replacementLength - replacedLength)]; + int inStart = 0, outStart = 0; + for (int i = 0; i < occurrenceCount; i++) { + int offset = starts[i] - inStart; + System.arraycopy(array, inStart, result, outStart, offset); + inStart += offset; + outStart += offset; + System.arraycopy( + replacementChars, + 0, + result, + outStart, + replacementLength); + inStart += replacedLength; + outStart += replacementLength; + } + System.arraycopy(array, inStart, result, outStart, max - inStart); + return result; + } + + public static final char[][] subarray(char[][] array, int start, int end) { + if (end == -1) + end = array.length; + if (start > end) + return null; + if (start < 0) + return null; + if (end > array.length) + return null; + + char[][] result = new char[end - start][]; + System.arraycopy(array, start, result, 0, end - start); + return result; + } + + public static final char[] subarray(char[] array, int start, int end) { + if (end == -1) + end = array.length; + if (start > end) + return null; + if (start < 0) + return null; + if (end > array.length) + return null; + + char[] result = new char[end - start]; + System.arraycopy(array, start, result, 0, end - start); + return result; + } + + public static final int indexOf(char toBeFound, char[] array) { + for (int i = 0; i < array.length; i++) + if (toBeFound == array[i]) + return i; + return -1; + } + final static public char[] trim(char[] chars) { + + if (chars == null) + return null; + + int start = 0, length = chars.length, end = length - 1; + while (start < length && chars[start] == ' ') { + start++; + } + while (end > start && chars[end] == ' ') { + end--; + } + if (start != 0 || end != length - 1) { + return subarray(chars, start, end + 1); + } + return chars; + } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java index a8741d4d89b..dca267a9497 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java @@ -11,6 +11,7 @@ package org.eclipse.cdt.internal.core.parser.scanner2; import java.io.PrintStream; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -1169,11 +1170,13 @@ public class Scanner2 implements IScanner, IScannerData { int textstart = bufferPos[bufferStackPos] + 1; int textend = textstart - 1; + boolean encounteredMultilineComment = false; while (bufferPos[bufferStackPos] + 1 < limit && buffer[bufferPos[bufferStackPos] + 1] != '\n') { skipOverNonWhiteSpace(); textend = bufferPos[bufferStackPos]; - skipOverWhiteSpace(); + if( skipOverWhiteSpace() ) + encounteredMultilineComment = true; } int textlen = textend - textstart + 1; @@ -1182,6 +1185,11 @@ public class Scanner2 implements IScanner, IScannerData { text = new char[textlen]; System.arraycopy(buffer, textstart, text, 0, textlen); } + + if( encounteredMultilineComment ) + text = removeMultilineCommentFromBuffer( text ); + if( CharArrayUtils.indexOf( '\n', text ) != -1 ) + text = removedEscapedNewline( text ); // Throw it in definitions.put(name, @@ -1190,6 +1198,48 @@ public class Scanner2 implements IScanner, IScannerData { : new FunctionStyleMacro(name, text, arglist)); } + /** + * @param text + * @return + */ + private char[] removedEscapedNewline(char[] text) { + char [] result = new char[ text.length ]; + Arrays.fill( result, ' '); + int counter = 0; + for( int i = 0; i < text.length; ++i ) + { + if( text[i] == '\\' && i+ 1 < text.length && text[i+1] == '\n' ) + ++i; + else + result[ counter++ ] = text[i]; + } + return CharArrayUtils.trim( result ); + } + + /** + * @param text + * @return + */ + private char[] removeMultilineCommentFromBuffer(char[] text) { + char [] result = new char[ text.length ]; + Arrays.fill( result, ' '); + int resultCount = 0; + for( int i = 0; i < text.length; ++i ) + { + if( text[i] == '/' && ( i+1 < text.length ) && text[i+1] == '*') + { + i += 2; + while( i < text.length && text[i] != '*' && i+1 < text.length && text[i+1] != '/') + ++i; + ++i; + } + else + result[resultCount++] = text[i]; + + } + return CharArrayUtils.trim( result ); + } + private void handlePPUndef() { char[] buffer = bufferStack[bufferStackPos]; int limit = bufferLimit[bufferStackPos]; @@ -1346,10 +1396,11 @@ public class Scanner2 implements IScanner, IScannerData { } } - private void skipOverWhiteSpace() { + private boolean skipOverWhiteSpace() { char[] buffer = bufferStack[bufferStackPos]; int limit = bufferLimit[bufferStackPos]; + boolean encounteredMultiLineComment = false; while (++bufferPos[bufferStackPos] < limit) { int pos = bufferPos[bufferStackPos]; switch (buffer[pos]) { @@ -1364,7 +1415,7 @@ public class Scanner2 implements IScanner, IScannerData { skipToNewLine(); // leave the new line there --bufferPos[bufferStackPos]; - return; + return false; } else if (buffer[pos + 1] == '*') { // C comment, find closing */ for (bufferPos[bufferStackPos] += 2; @@ -1375,6 +1426,7 @@ public class Scanner2 implements IScanner, IScannerData { && pos + 1 < limit && buffer[pos + 1] == '/') { ++bufferPos[bufferStackPos]; + encounteredMultiLineComment = true; break; } } @@ -1393,8 +1445,9 @@ public class Scanner2 implements IScanner, IScannerData { // fell out of switch without continuing, we're done --bufferPos[bufferStackPos]; - return; + return encounteredMultiLineComment; } + return encounteredMultiLineComment; } private void skipOverNonWhiteSpace() { @@ -1409,8 +1462,17 @@ public class Scanner2 implements IScanner, IScannerData { case '\n': --bufferPos[bufferStackPos]; return; - case '\\': + case '/': int pos = bufferPos[bufferStackPos]; + if( pos +1 < limit && ( buffer[pos+1] == '/' ) || ( buffer[pos+1] == '*') ) + { + --bufferPos[bufferStackPos]; + return; + } + break; + + case '\\': + pos = bufferPos[bufferStackPos]; if (pos + 1 < limit && buffer[pos + 1] == '\n') { // \n is whitespace --bufferPos[bufferStackPos]; @@ -1418,7 +1480,9 @@ public class Scanner2 implements IScanner, IScannerData { } break; case '"': - boolean escaped = false; + boolean escaped = false; + if( bufferPos[bufferStackPos] -1 > 0 && buffer[bufferPos[bufferStackPos] -1 ] == '\\' ) + escaped = true; loop: while (++bufferPos[bufferStackPos] < bufferLimit[bufferStackPos]) { switch (buffer[bufferPos[bufferStackPos]]) { @@ -1431,6 +1495,18 @@ public class Scanner2 implements IScanner, IScannerData { continue; } break loop; + case '\n': + if( !escaped ) + break loop; + case '/': + if( escaped && ( bufferPos[bufferStackPos] +1 < limit ) && + ( buffer[bufferPos[ bufferStackPos ] + 1] == '/' || + buffer[bufferPos[ bufferStackPos ] + 1] == '*' ) ) + { + --bufferPos[bufferStackPos]; + return; + } + default: escaped = false; }