Follow-up fix for 177154: [patch] single-line comments incorrectly retained in macros (junit test case by Ed Swartz <ed.swartz@nokia.com>)

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2Tests.java

@@ -3648,4 +3648,35 @@ public class AST2Tests extends AST2BaseTest {
     	for (int i = 0; i < LANGUAGES.length; i++)
     		parse( buffer.toString(), LANGUAGES[i], true, true );
     }
+    
+	public void testMacroCommentsBug_177154_2() throws Exception {
+		String noCommentMacro =
+			"#define Sonar16G(x) ((Sonr16G(x)<<16)|0xff000000L)\r\n"; 
+		String commentMacro = 	
+			"#define Sonar16G(x) ((Sonr16G(x)<<16)|0xff000000L)	// add the varf value\r\n"; 
+
+		String textTail = "\r\n" + 
+				"const int snd16SonarR[32] = {\r\n" + 
+				"		0xFF000000L,   Sonar16G(0x01), Sonar16G(0x02), Sonar16G(0x03),\r\n" + 
+				"		Sonar16G(0x04), Sonar16G(0x05), Sonar16G(0x06), Sonar16G(0x07),\r\n" + 
+				"		Sonar16G(0x08), Sonar16G(0x09), Sonar16G(0x0A), Sonar16G(0x0B),\r\n" + 
+				"		Sonar16G(0x0C), Sonar16G(0x0D), Sonar16G(0x0E), Sonar16G(0x0F),\r\n" + 
+				"		Sonar16G(0x10), Sonar16G(0x11), Sonar16G(0x12), Sonar16G(0x13),\r\n" + 
+				"		Sonar16G(0x14), Sonar16G(0x15), Sonar16G(0x16), Sonar16G(0x17),\r\n" + 
+				"		Sonar16G(0x18), Sonar16G(0x19), Sonar16G(0x1A), Sonar16G(0x1B),\r\n" + 
+				"		Sonar16G(0x1C), Sonar16G(0x1D), Sonar16G(0x1E), Sonar16G(0x1F),\r\n" + 
+				"	};\r\n" + 
+				"\r\n" + 
+				"";
+		
+		// this should work
+		String textNoComment = noCommentMacro + textTail;
+		IASTTranslationUnit tu = parse( textNoComment, ParserLanguage.CPP, true, true );
+
+		// this fails
+		String textComment = commentMacro + textTail;
+		tu = parse( textComment, ParserLanguage.CPP, true, true );
+		
+	}
+        
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/BaseScanner.java

@@ -2330,8 +2330,9 @@ abstract class BaseScanner implements IScanner {
     }
 
     /**
-     * @param text
+     * Remove line and block comments from the given char array.
-     * @return
+     * @param text  the char array
+     * @return a char array without comment
      */
     protected char[] removeCommentFromBuffer(char[] text) {
         char[] result = new char[text.length];
@@ -2341,37 +2342,52 @@ abstract class BaseScanner implements IScanner {
         boolean insideSingleQuote= false;
         boolean escaped= false;
         // either a single-line or multi-line comment was found
-        for (int i = 0; i < text.length; ++i) {
+        forLoop: for (int i = 0; i < text.length; ++i) {
-            if (!insideString && !insideSingleQuote && (text[i] == '/' 
+            final char c= text[i];
-            		&& (i + 1 < text.length) && (text[i + 1] == '*' || text[i + 1] == '/'))) {
+        	switch (c) {
-            	if (text[i + 1] == '/') {
+        	case '/':
-            		// done
+    			if (!insideString && !insideSingleQuote && i + 1 < text.length) {
-            		break;
+    				final char c2= text[i + 1];
-            	} else {
+    				if (c2 == '/') {
-	                i += 2;
+    					// done
-	                while (i < text.length
+    					break forLoop;
-	                        && !(text[i] == '*' && i + 1 < text.length && text[i + 1] == '/'))
+    				} else if (c2 == '*') {
-	                    ++i;
+    					i += 2;
+    					while (i < text.length
+    							&& !(text[i] == '*' && i + 1 < text.length && text[i + 1] == '/'))
+    						++i;
+    					++i;
+    					continue;
+    				}
+    			}
+    			escaped= false;
+                break;
+            case '\\':
+           		escaped = !escaped;
+                break;
+            case '"':
+            	if (!insideSingleQuote) {
+            		insideString= insideString ? escaped : true;
             	}
-                ++i;
+            	escaped= false;
-            } else {
+            	break;
-            	switch (text[i]) {
+            case '\'':
-                case '\\':
+            	if (!insideString) {
-                    escaped = !escaped;
+            		insideSingleQuote= insideSingleQuote ? escaped : true;
-                    break;
-                case '"':
-                	if (!insideSingleQuote && !escaped) {
-                		insideString= !insideString;
-                	}
-                	break;
-                case '\'':
-                	if (!insideString && !escaped) {
-                		insideSingleQuote= !insideSingleQuote;
-                	}
-                	break;
             	}
-                result[resultCount++] = text[i];
+            	escaped= false;
-            }
+            	break;
+            case '\t':
+            	if (!insideString && !insideSingleQuote) {
+            		result[resultCount++]= ' ';
+                	continue;
+            	}
+        		escaped= false;
+            	break;
+        	default:
+        		escaped= false;
+        	}
+            result[resultCount++] = c;
         }
         return CharArrayUtils.trim(result);
     }
@@ -3042,15 +3058,15 @@ abstract class BaseScanner implements IScanner {
                 escaped = !escaped;
                 continue;
             case '"': 
-            	if (!insideComment && !insideSingleQuote && !escaped) {
+            	if (!insideComment && !insideSingleQuote) {
-            		insideString= !insideString;
+            		insideString= insideString ? escaped : true;
             	}
-            	continue;
+            	break;
             case '\'':
-            	if (!insideComment && !insideString && !escaped) {
+            	if (!insideComment && !insideString) {
-            		insideSingleQuote= !insideSingleQuote;
+            		insideSingleQuote= insideSingleQuote ? escaped : true;
             	}
-            	continue;
+            	break;
             case '\n':
                 if (escaped) {
                     break;