Fix for 124113: Partition Scanner incorrectly handles line splicing

core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/AsmPartitionerTest.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2007 IBM Corporation and others.
+ * Copyright (c) 2000, 2008 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -1006,55 +1006,67 @@ public class AsmPartitionerTest extends TestCase {
 			fDocument.replace(2, 0, "\\\\");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  29,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  23,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(23, 5, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(28, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
-//			checkPartitioning(expectation, result);
+			checkPartitioning(expectation, result);
 
 			// replace one backslash with a newline
 			fDocument.replace(3, 1, "\n");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  29,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  23,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(23, 5, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(28, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
-//			checkPartitioning(expectation, result);
+			checkPartitioning(expectation, result);
 
 			// insert backslash and newline inside multiline comment
 			fDocument.replace(26, 0, "\\\r\n");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  32,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  23,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(23, 8, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(31, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(32, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
-//			checkPartitioning(expectation, result);
+			checkPartitioning(expectation, result);
 
 			// delete NL leaving only CR
 			fDocument.replace(28, 1, "");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  31,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  23,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(23, 7, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(30, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(31, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
-//			checkPartitioning(expectation, result);
+			checkPartitioning(expectation, result);
 
 			// delete backslash
 			fDocument.replace(26, 1, "");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  30,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  23,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(23, 6, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(29, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(30, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
-//			checkPartitioning(expectation, result);
+			checkPartitioning(expectation, result);
 
 			// insert white space before #
 			fDocument.replace(0, 0, "  \t");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  33,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  26,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(26, 6, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(32, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(33, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
-//			checkPartitioning(expectation, result);
+			checkPartitioning(expectation, result);
 
 		} catch (BadLocationException x) {
 			assertTrue(false);
@@ -1195,4 +1207,36 @@ public class AsmPartitionerTest extends TestCase {
 		checkPartitioning(expectation, result);
 	}
 
+	public void testLineSplicing_Bug124113() {
+		try {
+
+			fDocument.replace(0, fDocument.getLength(), "* comment... \\\\\ncontinued");
+
+			ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
+			TypedRegion[] expectation= {
+				new TypedRegion(0,  fDocument.getLength(),  ICPartitions.C_SINGLE_LINE_COMMENT)
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(0, fDocument.getLength(), "#define D \\\\\ncontinued");
+
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  fDocument.getLength(),  ICPartitions.C_PREPROCESSOR)
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(0, fDocument.getLength(), "\"str\\\\\ncontinued\"");
+
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  fDocument.getLength(),  ICPartitions.C_STRING)
+			};
+			checkPartitioning(expectation, result);
+
+		} catch (BadLocationException x) {
+			assertTrue(false);
+		}
+	}
+
 }

core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/CPartitionerTest.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2006 IBM Corporation and others.
+ * Copyright (c) 2000, 2008 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -993,7 +993,7 @@ public class CPartitionerTest extends TestCase {
 			fDocument.replace(fDocument.getLength(), 0, "#");
 			fDocument.replace(fDocument.getLength(), 0, " ");
 			fDocument.replace(fDocument.getLength(), 0, "\t");
-			fDocument.replace(fDocument.getLength(), 0, "include <float.h> /* */");
+			fDocument.replace(fDocument.getLength(), 0, "include <float.h> ");
 
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
@@ -1001,10 +1001,28 @@ public class CPartitionerTest extends TestCase {
 			};
 			checkPartitioning(expectation, result);
 
+			fDocument.replace(fDocument.getLength(), 0, "//   ");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  21,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(21, 5, ICPartitions.C_SINGLE_LINE_COMMENT),
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(21, 5, "/* */");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  21,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(21, 5, ICPartitions.C_MULTI_LINE_COMMENT),
+			};
+			checkPartitioning(expectation, result);
+
 			fDocument.replace(fDocument.getLength(), 0, "\nz");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  27,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  21,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(21, 5, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(26, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(27, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
 			checkPartitioning(expectation, result);
@@ -1013,7 +1031,9 @@ public class CPartitionerTest extends TestCase {
 			fDocument.replace(2, 0, "\\\\");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  29,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  23,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(23, 5, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(28, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
 			checkPartitioning(expectation, result);
@@ -1022,7 +1042,9 @@ public class CPartitionerTest extends TestCase {
 			fDocument.replace(3, 1, "\n");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  29,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  23,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(23, 5, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(28, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
 			checkPartitioning(expectation, result);
@@ -1031,7 +1053,9 @@ public class CPartitionerTest extends TestCase {
 			fDocument.replace(26, 0, "\\\r\n");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  32,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  23,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(23, 8, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(31, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(32, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
 			checkPartitioning(expectation, result);
@@ -1040,7 +1064,9 @@ public class CPartitionerTest extends TestCase {
 			fDocument.replace(28, 1, "");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  31,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  23,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(23, 7, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(30, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(31, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
 			checkPartitioning(expectation, result);
@@ -1049,7 +1075,9 @@ public class CPartitionerTest extends TestCase {
 			fDocument.replace(26, 1, "");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  30,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  23,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(23, 6, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(29, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(30, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
 			checkPartitioning(expectation, result);
@@ -1058,7 +1086,9 @@ public class CPartitionerTest extends TestCase {
 			fDocument.replace(0, 0, "  \t");
 			result= fDocument.computePartitioning(0, fDocument.getLength());
 			expectation= new TypedRegion[] {
-				new TypedRegion(0,  33,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(0,  26,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(26, 6, ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(32, 1,  ICPartitions.C_PREPROCESSOR),
 				new TypedRegion(33, 1, IDocument.DEFAULT_CONTENT_TYPE)
 			};
 			checkPartitioning(expectation, result);
@@ -1068,4 +1098,67 @@ public class CPartitionerTest extends TestCase {
 		}
 	}
 
+	public void testLineSplicing_Bug124113() {
+		try {
+
+			fDocument.replace(0, fDocument.getLength(), "// comment... \\\\\ncontinued");
+
+			ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
+			TypedRegion[] expectation= {
+				new TypedRegion(0,  fDocument.getLength(),  ICPartitions.C_SINGLE_LINE_COMMENT)
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(0, fDocument.getLength(), "#define D \\\\\ncontinued");
+
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  fDocument.getLength(),  ICPartitions.C_PREPROCESSOR)
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(0, fDocument.getLength(), "\"str\\\\\ncontinued\"");
+
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  fDocument.getLength(),  ICPartitions.C_STRING)
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(0, fDocument.getLength(), "'\\\\\nc'");
+
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  fDocument.getLength(),  ICPartitions.C_CHARACTER)
+			};
+			checkPartitioning(expectation, result);
+
+		} catch (BadLocationException x) {
+			assertTrue(false);
+		}
+	}
+
+	public void testCommentInPreprocessorString() {
+		try {
+
+			fDocument.replace(0, fDocument.getLength(), "#define S \"http://www.foo.bar\"");
+
+			ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
+			TypedRegion[] expectation= {
+				new TypedRegion(0,  fDocument.getLength(),  ICPartitions.C_PREPROCESSOR)
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(0, fDocument.getLength(), "#define S \"http:/* */\"");
+
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  fDocument.getLength(),  ICPartitions.C_PREPROCESSOR)
+			};
+			checkPartitioning(expectation, result);
+
+		} catch (BadLocationException x) {
+			assertTrue(false);
+		}
+	}
 }

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/CPreprocessorScanner.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2007 Wind River Systems, Inc. and others.
+ * Copyright (c) 2007, 2008 Wind River Systems, Inc. and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -14,9 +14,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.eclipse.jface.preference.IPreferenceStore;
-import org.eclipse.jface.text.rules.EndOfLineRule;
 import org.eclipse.jface.text.rules.IRule;
-import org.eclipse.jface.text.rules.MultiLineRule;
 import org.eclipse.jface.text.rules.PatternRule;
 import org.eclipse.jface.text.rules.Token;
 import org.eclipse.jface.text.rules.WhitespaceRule;
@@ -36,8 +34,6 @@ public class CPreprocessorScanner extends AbstractCScanner {
 
     /** Properties for tokens. */
 	private static String[] fgTokenProperties= {
-		ICColorConstants.C_SINGLE_LINE_COMMENT,
-		ICColorConstants.C_MULTI_LINE_COMMENT,
 		ICColorConstants.C_KEYWORD,
 		ICColorConstants.PP_DIRECTIVE,
 		ICColorConstants.PP_DEFAULT,
@@ -103,14 +99,6 @@ public class CPreprocessorScanner extends AbstractCScanner {
         CHeaderRule headerRule = new CHeaderRule(token);
         rules.add(headerRule);
 
-        token = getToken(ICColorConstants.C_SINGLE_LINE_COMMENT);
-        IRule lineCommentRule = new EndOfLineRule("//", token, '\\', true); //$NON-NLS-1$
-        rules.add(lineCommentRule);
-
-        token = getToken(ICColorConstants.C_MULTI_LINE_COMMENT);
-        IRule blockCommentRule = new MultiLineRule("/*", "*/", token, '\\'); //$NON-NLS-1$ //$NON-NLS-2$
-        rules.add(blockCommentRule);
-
         token = getToken(ICColorConstants.C_STRING);
         IRule stringRule = new PatternRule("\"", "\"", token, '\\', true, true, true); //$NON-NLS-1$ //$NON-NLS-2$
         rules.add(stringRule);

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/FastCPartitionScanner.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2006 IBM Corporation and others.
+ * Copyright (c) 2000, 2008 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -37,6 +37,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	private static final int STRING= 4;
 	private static final int PREPROCESSOR= 5;
 	private static final int PREPROCESSOR_MULTI_LINE_COMMENT= 6;
+	private static final int PREPROCESSOR_STRING= 7;
 	
 	// beginning of prefixes and postfixes
 	private static final int NONE= 0;
@@ -46,6 +47,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	private static final int STAR= 4; // postfix for MULTI_LINE_COMMENT
 	private static final int CARRIAGE_RETURN=5; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
 	private static final int BACKSLASH_CR= 6; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
+	private static final int BACKSLASH_BACKSLASH= 7; // postfix for STRING, CHARACTER
 	
 	/** The scanner. */
 	private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000);	// faster implementation
@@ -65,7 +67,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	private boolean fFirstCharOnLine= true;
 
 	// emulate CPartitionScanner
-	private boolean fEmulate= false;
+	private final boolean fEmulate;
 	private int fCCodeOffset;
 	private int fCCodeLength;
 	
@@ -76,7 +78,8 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 		new Token(C_CHARACTER),
 		new Token(C_STRING),
 		new Token(C_PREPROCESSOR),
-		new Token(C_PREPROCESSOR)
+		new Token(C_MULTI_LINE_COMMENT),
+		new Token(C_PREPROCESSOR),
 	};
 
 	public FastCPartitionScanner(boolean emulate) {
@@ -128,7 +131,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 
 	 		case '\r':
 	 			fFirstCharOnLine= true;
-	 			if (!fEmulate && fLast == BACKSLASH) {
+	 			if (!fEmulate && fLast == BACKSLASH || fLast == BACKSLASH_BACKSLASH) {
 	 				fLast= BACKSLASH_CR;
 					fTokenLength++;
  					continue;
@@ -171,11 +174,18 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	 			}
 
 	 		case '\\':
-	 			if (fLast == BACKSLASH) {
+				switch (fState) {
-	 				consume();
+				case CHARACTER:
+				case STRING:
+				case PREPROCESSOR_STRING:
+					fTokenLength++;
+					fLast= fLast == BACKSLASH ? BACKSLASH_BACKSLASH : BACKSLASH;
+					continue;
+				default:
+					fTokenLength++;
+					fLast= BACKSLASH;
 					continue;
 	 			}
-	 			break;
 
 	 		case '\n':
 	 			fFirstCharOnLine= true;
@@ -184,9 +194,10 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 				case CHARACTER:
 				case STRING:
 				case PREPROCESSOR:
+				case PREPROCESSOR_STRING:
 					// assert(fTokenLength > 0);
-					// if last char was a backslash then we have an escaped line
+					// if last char was a backslash then we have spliced line
-					if (fLast != BACKSLASH && fLast != BACKSLASH_CR) {
+					if (fLast != BACKSLASH && fLast != BACKSLASH_CR && fLast != BACKSLASH_BACKSLASH) {
 						return postFix(fState);
 					}
 
@@ -202,6 +213,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 					case CHARACTER:
 					case STRING:
 					case PREPROCESSOR:
+					case PREPROCESSOR_STRING:
 
 						int last;
 						int newState;
@@ -352,10 +364,44 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 				break;
 
 	 		case SINGLE_LINE_COMMENT:
+				consume();
+ 				break;
+
+	 		case PREPROCESSOR:
 	 			switch (ch) {
-	 			case '\\':
+				case '/':
-					fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
+					if (fLast == SLASH) {
-					fTokenLength++;
+						if (fTokenLength - getLastLength(fLast) > 0) {
+							return preFix(fState, SINGLE_LINE_COMMENT, SLASH, 2);
+						} else {
+							preFix(fState, SINGLE_LINE_COMMENT, SLASH, 2);
+							fTokenOffset += fTokenLength;
+							fTokenLength= fPrefixLength;
+							break;
+						}
+					} else {
+						consume();
+						fLast= SLASH;
+					}
+					break;
+	
+				case '*':
+					if (fLast == SLASH) {
+						if (fTokenLength - getLastLength(fLast) > 0) {
+							return preFix(fState, PREPROCESSOR_MULTI_LINE_COMMENT, SLASH_STAR, 2);
+						} else {
+							preFix(fState, PREPROCESSOR_MULTI_LINE_COMMENT, SLASH_STAR, 2);
+							fTokenOffset += fTokenLength;
+							fTokenLength= fPrefixLength;
+							break;
+						}
+					}
+					
+				case '"':
+					if (fLast != BACKSLASH) {
+						fState= PREPROCESSOR_STRING;
+					}
+					consume();
 					break;
 					
 		 		default:
@@ -364,32 +410,14 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	 			}
 	 			break;
 
-	 		case PREPROCESSOR:
+	 		case PREPROCESSOR_STRING:
 				switch (ch) {
-	 			case '\\':
+				case '"':
-					fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
+					if (fLast != BACKSLASH) {
-					fTokenLength++;
+						fState= PREPROCESSOR;
-					break;
-
-				case '/':
-					if (fLast == SLASH) {
-						consume();
-						break;
-					} else {
-						fTokenLength++;
-						fLast= SLASH;
-						break;
 					}
-	
-				case '*':
-					if (fLast == SLASH) {
-						fState= PREPROCESSOR_MULTI_LINE_COMMENT;
 					consume();
 					break;
-					} else {
-						consume();
-						break;
-					}
 					
 		 		default:
 					consume();
@@ -406,7 +434,9 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	
 				case '/':
 					if (fLast == STAR) {
+						IToken token= postFix(fState);
 						fState= PREPROCESSOR;
+						return token;
 					}
 					consume();
 					break;
@@ -440,11 +470,6 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 				
 	 		case STRING:
 	 			switch (ch) {
-	 			case '\\':
-					fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
-					fTokenLength++;
-					break;
-					
 				case '\"':	 			 			
 	 				if (fLast != BACKSLASH) {
 	 					return postFix(STRING);
@@ -462,11 +487,6 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	
 	 		case CHARACTER:
 	 			switch (ch) {
-				case '\\':
-					fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
-					fTokenLength++;
-					break;
-	
 	 			case '\'':
 	 				if (fLast != BACKSLASH) {
 	 					return postFix(CHARACTER);
@@ -501,6 +521,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 
 		case SLASH_STAR:
 		case BACKSLASH_CR:
+		case BACKSLASH_BACKSLASH:
 			return 2;
 
 		}	

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/asm/AsmPartitionScanner.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2007 IBM Corporation and others.
+ * Copyright (c) 2000, 2008 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -71,6 +71,7 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
 	private static final int STAR= 4; // postfix for MULTI_LINE_COMMENT
 	private static final int CARRIAGE_RETURN=5; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
 	private static final int BACKSLASH_CR= 6; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
+	private static final int BACKSLASH_BACKSLASH= 7; // postfix for STRING, CHARACTER
 	
 	/** The scanner. */
 	private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000);
@@ -185,7 +186,7 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
 
 	 		case '\r':
 	 			fFirstCharOnLine= true;
-	 			if (fLast == BACKSLASH) {
+	 			if (fLast == BACKSLASH || fLast == BACKSLASH_BACKSLASH) {
 	 				fLast= BACKSLASH_CR;
 					fTokenLength++;
  					continue;
@@ -221,11 +222,17 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
 	 			}
 
 	 		case '\\':
-	 			if (fLast == BACKSLASH) {
+				switch (fState) {
-	 				consume();
+				case CHARACTER:
+				case STRING:
+					fTokenLength++;
+					fLast= fLast == BACKSLASH ? BACKSLASH_BACKSLASH : BACKSLASH;
+					continue;
+				default:
+					fTokenLength++;
+					fLast= BACKSLASH;
 					continue;
 	 			}
-	 			break;
 
 	 		case '\n':
 	 			fFirstCharOnLine= true;
@@ -236,7 +243,7 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
 				case PREPROCESSOR:
 					// assert(fTokenLength > 0);
 					// if last char was a backslash then we have an escaped line
-					if (fLast != BACKSLASH && fLast != BACKSLASH_CR) {
+					if (fLast != BACKSLASH && fLast != BACKSLASH_CR && fLast != BACKSLASH_BACKSLASH) {
 						return postFix(fState);
 					}
 
@@ -431,25 +438,11 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
 				break;
 
 	 		case SINGLE_LINE_COMMENT:
-	 			switch (ch) {
-	 			case '\\':
-					fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
-					fTokenLength++;
-					break;
-					
-		 		default:
 				consume();
  				break;
-	 			}
-	 			break;
 
 	 		case PREPROCESSOR:
 	 			switch (ch) {
-	 			case '\\':
-					fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
-					fTokenLength++;
-					break;
-
 				case '/':
 					if (fLast == SLASH) {
 						consume();
@@ -525,11 +518,6 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
 				
 	 		case STRING:
 	 			switch (ch) {
-	 			case '\\':
-					fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
-					fTokenLength++;
-					break;
-					
 				case '\"':
 	 				if (fLast != BACKSLASH) {
 	 					return postFix(STRING);
@@ -579,6 +567,7 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
 
 		case SLASH_STAR:
 		case BACKSLASH_CR:
+		case BACKSLASH_BACKSLASH:
 			return 2;
 
 		}