Further progress w/Scanner2.

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java

@@ -278,7 +278,7 @@ public class Scanner2Test extends BaseScanner2Test
 
 		initializeScanner("#define MULTICOMMENT X /* comment1 */ + Y /* comment 2 */"); //$NON-NLS-1$
 		validateEOF();
-		validateDefinition("MULTICOMMENT", "X /* comment1 */ + Y"); //$NON-NLS-1$ //$NON-NLS-2$
+		validateDefinition("MULTICOMMENT", "X  + Y"); //$NON-NLS-1$ //$NON-NLS-2$
 
 		initializeScanner("#define SIMPLE_STRING This is a simple string.\n"); //$NON-NLS-1$
 		validateEOF();

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayUtils.java

@@ -71,4 +71,129 @@ public class CharArrayUtils {
 		System.arraycopy(str, start, copy, 0, length);
 		return copy;
 	}
+	
+	public static final char[] concat(char[] first, char[] second) {
+		if (first == null)
+			return second;
+		if (second == null)
+			return first;
+
+		int length1 = first.length;
+		int length2 = second.length;
+		char[] result = new char[length1 + length2];
+		System.arraycopy(first, 0, result, 0, length1);
+		System.arraycopy(second, 0, result, length1, length2);
+		return result;
+	}
+	
+	public static final char[] replace(
+		char[] array,
+		char[] toBeReplaced,
+		char[] replacementChars) {
+
+		int max = array.length;
+		int replacedLength = toBeReplaced.length;
+		int replacementLength = replacementChars.length;
+
+		int[] starts = new int[5];
+		int occurrenceCount = 0;
+
+		if (!equals(toBeReplaced, replacementChars)) {
+
+			next : for (int i = 0; i < max; i++) {
+				int j = 0;
+				while (j < replacedLength) {
+					if (i + j == max)
+						continue next;
+					if (array[i + j] != toBeReplaced[j++])
+						continue next;
+				}
+				if (occurrenceCount == starts.length) {
+					System.arraycopy(
+						starts,
+						0,
+						starts = new int[occurrenceCount * 2],
+						0,
+						occurrenceCount);
+				}
+				starts[occurrenceCount++] = i;
+			}
+		}
+		if (occurrenceCount == 0)
+			return array;
+		char[] result =
+			new char[max
+				+ occurrenceCount * (replacementLength - replacedLength)];
+		int inStart = 0, outStart = 0;
+		for (int i = 0; i < occurrenceCount; i++) {
+			int offset = starts[i] - inStart;
+			System.arraycopy(array, inStart, result, outStart, offset);
+			inStart += offset;
+			outStart += offset;
+			System.arraycopy(
+				replacementChars,
+				0,
+				result,
+				outStart,
+				replacementLength);
+			inStart += replacedLength;
+			outStart += replacementLength;
+		}
+		System.arraycopy(array, inStart, result, outStart, max - inStart);
+		return result;
+	}
+
+	public static final char[][] subarray(char[][] array, int start, int end) {
+		if (end == -1)
+			end = array.length;
+		if (start > end)
+			return null;
+		if (start < 0)
+			return null;
+		if (end > array.length)
+			return null;
+
+		char[][] result = new char[end - start][];
+		System.arraycopy(array, start, result, 0, end - start);
+		return result;
+	}
+
+	public static final char[] subarray(char[] array, int start, int end) {
+		if (end == -1)
+			end = array.length;
+		if (start > end)
+			return null;
+		if (start < 0)
+			return null;
+		if (end > array.length)
+			return null;
+
+		char[] result = new char[end - start];
+		System.arraycopy(array, start, result, 0, end - start);
+		return result;
+	}
+	
+	public static final int indexOf(char toBeFound, char[] array) {
+		for (int i = 0; i < array.length; i++)
+			if (toBeFound == array[i])
+				return i;
+		return -1;
+	}
+	final static public char[] trim(char[] chars) {
+
+		if (chars == null)
+			return null;
+
+		int start = 0, length = chars.length, end = length - 1;
+		while (start < length && chars[start] == ' ') {
+			start++;
+		}
+		while (end > start && chars[end] == ' ') {
+			end--;
+		}
+		if (start != 0 || end != length - 1) {
+			return subarray(chars, start, end + 1);
+		}
+		return chars;
+	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java

@@ -11,6 +11,7 @@
 package org.eclipse.cdt.internal.core.parser.scanner2;
 
 import java.io.PrintStream;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -1169,11 +1170,13 @@ public class Scanner2 implements IScanner, IScannerData {
 		int textstart = bufferPos[bufferStackPos] + 1;
 		int textend = textstart - 1;
 		
+		boolean encounteredMultilineComment = false;
 		while (bufferPos[bufferStackPos] + 1 < limit
 				&& buffer[bufferPos[bufferStackPos] + 1] != '\n') {
 			skipOverNonWhiteSpace();
 			textend = bufferPos[bufferStackPos];
-			skipOverWhiteSpace();
+			if( skipOverWhiteSpace() )
+				encounteredMultilineComment = true;
 		}
 
 		int textlen = textend - textstart + 1;
@@ -1183,6 +1186,11 @@ public class Scanner2 implements IScanner, IScannerData {
 			System.arraycopy(buffer, textstart, text, 0, textlen);
 		}
 		
+		if( encounteredMultilineComment )
+			text = removeMultilineCommentFromBuffer( text );
+		if( CharArrayUtils.indexOf( '\n', text ) != -1 )
+			text = removedEscapedNewline( text );
+			
 		// Throw it in
 		definitions.put(name,
 				arglist == null
@@ -1190,6 +1198,48 @@ public class Scanner2 implements IScanner, IScannerData {
 				: new FunctionStyleMacro(name, text, arglist));
 	}
 	
+	/**
+	 * @param text
+	 * @return
+	 */
+	private char[] removedEscapedNewline(char[] text) {
+		char [] result = new char[ text.length ];
+		Arrays.fill( result, ' ');
+		int counter = 0;
+		for( int i = 0;  i < text.length; ++i )
+		{
+			if( text[i] == '\\' && i+ 1 < text.length && text[i+1] == '\n' )
+				++i;
+			else
+				result[ counter++ ] = text[i];
+		}
+		return CharArrayUtils.trim( result );
+	}
+
+	/**
+	 * @param text
+	 * @return
+	 */
+	private char[] removeMultilineCommentFromBuffer(char[] text) {
+		char [] result = new char[ text.length ];
+		Arrays.fill( result, ' ');
+		int resultCount = 0;
+		for( int i = 0; i < text.length; ++i )
+		{
+			if( text[i] == '/' && ( i+1 < text.length ) && text[i+1] == '*')
+			{
+				i += 2;
+				while( i < text.length && text[i] != '*' && i+1 < text.length && text[i+1] != '/')
+					++i;
+				++i;
+			}
+			else
+				result[resultCount++] = text[i];
+				
+		}
+		return CharArrayUtils.trim( result );
+	}
+
 	private void handlePPUndef() {
 		char[] buffer = bufferStack[bufferStackPos];
 		int limit = bufferLimit[bufferStackPos];
@@ -1346,10 +1396,11 @@ public class Scanner2 implements IScanner, IScannerData {
 		}
 	}
 	
-	private void skipOverWhiteSpace() {
+	private boolean skipOverWhiteSpace() {
 		char[] buffer = bufferStack[bufferStackPos];
 		int limit = bufferLimit[bufferStackPos];
 		
+		boolean encounteredMultiLineComment = false;
 		while (++bufferPos[bufferStackPos] < limit) {
 			int pos = bufferPos[bufferStackPos];
 			switch (buffer[pos]) {
@@ -1364,7 +1415,7 @@ public class Scanner2 implements IScanner, IScannerData {
 							skipToNewLine();
 							// leave the new line there
 							--bufferPos[bufferStackPos];
-							return;
+							return false;
 						} else if (buffer[pos + 1] == '*') {
 							// C comment, find closing */
 							for (bufferPos[bufferStackPos] += 2;
@@ -1375,6 +1426,7 @@ public class Scanner2 implements IScanner, IScannerData {
 										&& pos + 1 < limit
 										&& buffer[pos + 1] == '/') {
 									++bufferPos[bufferStackPos];
+									encounteredMultiLineComment = true;
 									break;
 								}
 							}
@@ -1393,8 +1445,9 @@ public class Scanner2 implements IScanner, IScannerData {
 			
 			// fell out of switch without continuing, we're done
 			--bufferPos[bufferStackPos];
-			return;
+			return encounteredMultiLineComment;
 		}
+		return encounteredMultiLineComment;
 	}
 	
 	private void skipOverNonWhiteSpace() {
@@ -1409,8 +1462,17 @@ public class Scanner2 implements IScanner, IScannerData {
 				case '\n':
 					--bufferPos[bufferStackPos];
 					return;
-				case '\\':
+				case '/':
 					int pos = bufferPos[bufferStackPos];
+					if( pos +1 < limit && ( buffer[pos+1] == '/' ) || ( buffer[pos+1] == '*') )
+					{
+						--bufferPos[bufferStackPos];
+						return;
+					}
+					break;
+													
+				case '\\':
+					pos = bufferPos[bufferStackPos];
 					if (pos + 1 < limit && buffer[pos + 1] == '\n') {
 						// \n is whitespace
 						--bufferPos[bufferStackPos];
@@ -1419,6 +1481,8 @@ public class Scanner2 implements IScanner, IScannerData {
 					break;
 				case '"':
 					boolean escaped = false; 
+					if( bufferPos[bufferStackPos] -1  > 0 && buffer[bufferPos[bufferStackPos] -1 ] == '\\' )
+						escaped = true;
 					loop:
 					while (++bufferPos[bufferStackPos] < bufferLimit[bufferStackPos]) {
 						switch (buffer[bufferPos[bufferStackPos]]) {
@@ -1431,6 +1495,18 @@ public class Scanner2 implements IScanner, IScannerData {
 									continue;
 								}
 								break loop;
+							case '\n':
+								if( !escaped )
+									break loop;
+							case '/': 
+								if( escaped && ( bufferPos[bufferStackPos] +1 < limit ) && 
+										( buffer[bufferPos[ bufferStackPos ] + 1] == '/' ||
+										  buffer[bufferPos[ bufferStackPos ] + 1] == '*' ) )
+								{
+									--bufferPos[bufferStackPos];
+									return;
+								}
+						
 							default:
 								escaped = false;
 						}