mirrors/cdt
1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-04-29 19:45:01 +02:00
Code Issues Wiki Activity

Fix for 124113: Partition Scanner incorrectly handles line splicing

Browse source
This commit is contained in:
Anton Leherbauer 2008-01-21 13:58:34 +00:00
parent ec3283c69d
commit 4cc8dfd7b5
5 changed files with 245 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/AsmPartitionerTest.java
View file

@ -1,5 +1,5 @@
/******************************************************************************* /*******************************************************************************
* Copyright (c) 2000, 2007 IBM Corporation and others. * Copyright (c) 2000, 2008 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials * All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0 * are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at * which accompanies this distribution, and is available at
@ -1006,55 +1006,67 @@ public class AsmPartitionerTest extends TestCase {
fDocument.replace(2, 0, "\\\\"); fDocument.replace(2, 0, "\\\\");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 29, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 23, ICPartitions.C_PREPROCESSOR),
new TypedRegion(23, 5, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(28, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
// checkPartitioning(expectation, result); checkPartitioning(expectation, result);
// replace one backslash with a newline // replace one backslash with a newline
fDocument.replace(3, 1, "\n"); fDocument.replace(3, 1, "\n");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 29, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 23, ICPartitions.C_PREPROCESSOR),
new TypedRegion(23, 5, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(28, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
// checkPartitioning(expectation, result); checkPartitioning(expectation, result);
// insert backslash and newline inside multiline comment // insert backslash and newline inside multiline comment
fDocument.replace(26, 0, "\\\r\n"); fDocument.replace(26, 0, "\\\r\n");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 32, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 23, ICPartitions.C_PREPROCESSOR),
new TypedRegion(23, 8, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(31, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(32, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(32, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
// checkPartitioning(expectation, result); checkPartitioning(expectation, result);
// delete NL leaving only CR // delete NL leaving only CR
fDocument.replace(28, 1, ""); fDocument.replace(28, 1, "");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 31, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 23, ICPartitions.C_PREPROCESSOR),
new TypedRegion(23, 7, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(30, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(31, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(31, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
// checkPartitioning(expectation, result); checkPartitioning(expectation, result);
// delete backslash // delete backslash
fDocument.replace(26, 1, ""); fDocument.replace(26, 1, "");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 30, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 23, ICPartitions.C_PREPROCESSOR),
new TypedRegion(23, 6, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(29, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(30, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(30, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
// checkPartitioning(expectation, result); checkPartitioning(expectation, result);
// insert white space before # // insert white space before #
fDocument.replace(0, 0, " \t"); fDocument.replace(0, 0, " \t");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 33, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 26, ICPartitions.C_PREPROCESSOR),
new TypedRegion(26, 6, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(32, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(33, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(33, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
// checkPartitioning(expectation, result); checkPartitioning(expectation, result);
} catch (BadLocationException x) { } catch (BadLocationException x) {
assertTrue(false); assertTrue(false);
@ -1195,4 +1207,36 @@ public class AsmPartitionerTest extends TestCase {
checkPartitioning(expectation, result); checkPartitioning(expectation, result);
} }
public void testLineSplicing_Bug124113() {
try {
fDocument.replace(0, fDocument.getLength(), "* comment... \\\\\ncontinued");
ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
TypedRegion[] expectation= {
new TypedRegion(0, fDocument.getLength(), ICPartitions.C_SINGLE_LINE_COMMENT)
};
checkPartitioning(expectation, result);
fDocument.replace(0, fDocument.getLength(), "#define D \\\\\ncontinued");
result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] {
new TypedRegion(0, fDocument.getLength(), ICPartitions.C_PREPROCESSOR)
};
checkPartitioning(expectation, result);
fDocument.replace(0, fDocument.getLength(), "\"str\\\\\ncontinued\"");
result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] {
new TypedRegion(0, fDocument.getLength(), ICPartitions.C_STRING)
};
checkPartitioning(expectation, result);
} catch (BadLocationException x) {
assertTrue(false);
}
}
} }

111
core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/CPartitionerTest.java
View file

@ -1,5 +1,5 @@
/******************************************************************************* /*******************************************************************************
* Copyright (c) 2000, 2006 IBM Corporation and others. * Copyright (c) 2000, 2008 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials * All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0 * are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at * which accompanies this distribution, and is available at
@ -993,7 +993,7 @@ public class CPartitionerTest extends TestCase {
fDocument.replace(fDocument.getLength(), 0, "#"); fDocument.replace(fDocument.getLength(), 0, "#");
fDocument.replace(fDocument.getLength(), 0, " "); fDocument.replace(fDocument.getLength(), 0, " ");
fDocument.replace(fDocument.getLength(), 0, "\t"); fDocument.replace(fDocument.getLength(), 0, "\t");
fDocument.replace(fDocument.getLength(), 0, "include <float.h> /* */"); fDocument.replace(fDocument.getLength(), 0, "include <float.h> ");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
@ -1001,10 +1001,28 @@ public class CPartitionerTest extends TestCase {
}; };
checkPartitioning(expectation, result); checkPartitioning(expectation, result);
fDocument.replace(fDocument.getLength(), 0, "// ");
result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] {
new TypedRegion(0, 21, ICPartitions.C_PREPROCESSOR),
new TypedRegion(21, 5, ICPartitions.C_SINGLE_LINE_COMMENT),
};
checkPartitioning(expectation, result);
fDocument.replace(21, 5, "/* */");
result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] {
new TypedRegion(0, 21, ICPartitions.C_PREPROCESSOR),
new TypedRegion(21, 5, ICPartitions.C_MULTI_LINE_COMMENT),
};
checkPartitioning(expectation, result);
fDocument.replace(fDocument.getLength(), 0, "\nz"); fDocument.replace(fDocument.getLength(), 0, "\nz");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 27, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 21, ICPartitions.C_PREPROCESSOR),
new TypedRegion(21, 5, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(26, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(27, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(27, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
checkPartitioning(expectation, result); checkPartitioning(expectation, result);
@ -1013,7 +1031,9 @@ public class CPartitionerTest extends TestCase {
fDocument.replace(2, 0, "\\\\"); fDocument.replace(2, 0, "\\\\");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 29, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 23, ICPartitions.C_PREPROCESSOR),
new TypedRegion(23, 5, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(28, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
checkPartitioning(expectation, result); checkPartitioning(expectation, result);
@ -1022,7 +1042,9 @@ public class CPartitionerTest extends TestCase {
fDocument.replace(3, 1, "\n"); fDocument.replace(3, 1, "\n");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 29, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 23, ICPartitions.C_PREPROCESSOR),
new TypedRegion(23, 5, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(28, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(29, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
checkPartitioning(expectation, result); checkPartitioning(expectation, result);
@ -1031,7 +1053,9 @@ public class CPartitionerTest extends TestCase {
fDocument.replace(26, 0, "\\\r\n"); fDocument.replace(26, 0, "\\\r\n");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 32, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 23, ICPartitions.C_PREPROCESSOR),
new TypedRegion(23, 8, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(31, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(32, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(32, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
checkPartitioning(expectation, result); checkPartitioning(expectation, result);
@ -1040,7 +1064,9 @@ public class CPartitionerTest extends TestCase {
fDocument.replace(28, 1, ""); fDocument.replace(28, 1, "");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 31, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 23, ICPartitions.C_PREPROCESSOR),
new TypedRegion(23, 7, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(30, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(31, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(31, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
checkPartitioning(expectation, result); checkPartitioning(expectation, result);
@ -1049,7 +1075,9 @@ public class CPartitionerTest extends TestCase {
fDocument.replace(26, 1, ""); fDocument.replace(26, 1, "");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 30, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 23, ICPartitions.C_PREPROCESSOR),
new TypedRegion(23, 6, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(29, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(30, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(30, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
checkPartitioning(expectation, result); checkPartitioning(expectation, result);
@ -1058,7 +1086,9 @@ public class CPartitionerTest extends TestCase {
fDocument.replace(0, 0, " \t"); fDocument.replace(0, 0, " \t");
result= fDocument.computePartitioning(0, fDocument.getLength()); result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] { expectation= new TypedRegion[] {
new TypedRegion(0, 33, ICPartitions.C_PREPROCESSOR), new TypedRegion(0, 26, ICPartitions.C_PREPROCESSOR),
new TypedRegion(26, 6, ICPartitions.C_MULTI_LINE_COMMENT),
new TypedRegion(32, 1, ICPartitions.C_PREPROCESSOR),
new TypedRegion(33, 1, IDocument.DEFAULT_CONTENT_TYPE) new TypedRegion(33, 1, IDocument.DEFAULT_CONTENT_TYPE)
}; };
checkPartitioning(expectation, result); checkPartitioning(expectation, result);
@ -1068,4 +1098,67 @@ public class CPartitionerTest extends TestCase {
} }
} }
public void testLineSplicing_Bug124113() {
try {
fDocument.replace(0, fDocument.getLength(), "// comment... \\\\\ncontinued");
ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
TypedRegion[] expectation= {
new TypedRegion(0, fDocument.getLength(), ICPartitions.C_SINGLE_LINE_COMMENT)
};
checkPartitioning(expectation, result);
fDocument.replace(0, fDocument.getLength(), "#define D \\\\\ncontinued");
result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] {
new TypedRegion(0, fDocument.getLength(), ICPartitions.C_PREPROCESSOR)
};
checkPartitioning(expectation, result);
fDocument.replace(0, fDocument.getLength(), "\"str\\\\\ncontinued\"");
result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] {
new TypedRegion(0, fDocument.getLength(), ICPartitions.C_STRING)
};
checkPartitioning(expectation, result);
fDocument.replace(0, fDocument.getLength(), "'\\\\\nc'");
result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] {
new TypedRegion(0, fDocument.getLength(), ICPartitions.C_CHARACTER)
};
checkPartitioning(expectation, result);
} catch (BadLocationException x) {
assertTrue(false);
}
}
public void testCommentInPreprocessorString() {
try {
fDocument.replace(0, fDocument.getLength(), "#define S \"http://www.foo.bar\"");
ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
TypedRegion[] expectation= {
new TypedRegion(0, fDocument.getLength(), ICPartitions.C_PREPROCESSOR)
};
checkPartitioning(expectation, result);
fDocument.replace(0, fDocument.getLength(), "#define S \"http:/* */\"");
result= fDocument.computePartitioning(0, fDocument.getLength());
expectation= new TypedRegion[] {
new TypedRegion(0, fDocument.getLength(), ICPartitions.C_PREPROCESSOR)
};
checkPartitioning(expectation, result);
} catch (BadLocationException x) {
assertTrue(false);
}
}
} }

14
core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/CPreprocessorScanner.java
View file

@ -1,5 +1,5 @@
/******************************************************************************* /*******************************************************************************
* Copyright (c) 2007 Wind River Systems, Inc. and others. * Copyright (c) 2007, 2008 Wind River Systems, Inc. and others.
* All rights reserved. This program and the accompanying materials * All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0 * are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at * which accompanies this distribution, and is available at
@ -14,9 +14,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.eclipse.jface.preference.IPreferenceStore; import org.eclipse.jface.preference.IPreferenceStore;
import org.eclipse.jface.text.rules.EndOfLineRule;
import org.eclipse.jface.text.rules.IRule; import org.eclipse.jface.text.rules.IRule;
import org.eclipse.jface.text.rules.MultiLineRule;
import org.eclipse.jface.text.rules.PatternRule; import org.eclipse.jface.text.rules.PatternRule;
import org.eclipse.jface.text.rules.Token; import org.eclipse.jface.text.rules.Token;
import org.eclipse.jface.text.rules.WhitespaceRule; import org.eclipse.jface.text.rules.WhitespaceRule;
@ -36,8 +34,6 @@ public class CPreprocessorScanner extends AbstractCScanner {
/** Properties for tokens. */ /** Properties for tokens. */
private static String[] fgTokenProperties= { private static String[] fgTokenProperties= {
ICColorConstants.C_SINGLE_LINE_COMMENT,
ICColorConstants.C_MULTI_LINE_COMMENT,
ICColorConstants.C_KEYWORD, ICColorConstants.C_KEYWORD,
ICColorConstants.PP_DIRECTIVE, ICColorConstants.PP_DIRECTIVE,
ICColorConstants.PP_DEFAULT, ICColorConstants.PP_DEFAULT,
@ -103,14 +99,6 @@ public class CPreprocessorScanner extends AbstractCScanner {
CHeaderRule headerRule = new CHeaderRule(token); CHeaderRule headerRule = new CHeaderRule(token);
rules.add(headerRule); rules.add(headerRule);
token = getToken(ICColorConstants.C_SINGLE_LINE_COMMENT);
IRule lineCommentRule = new EndOfLineRule("//", token, '\\', true); //$NON-NLS-1$
rules.add(lineCommentRule);
token = getToken(ICColorConstants.C_MULTI_LINE_COMMENT);
IRule blockCommentRule = new MultiLineRule("/*", "*/", token, '\\'); //$NON-NLS-1$ //$NON-NLS-2$
rules.add(blockCommentRule);
token = getToken(ICColorConstants.C_STRING); token = getToken(ICColorConstants.C_STRING);
IRule stringRule = new PatternRule("\"", "\"", token, '\\', true, true, true); //$NON-NLS-1$ //$NON-NLS-2$ IRule stringRule = new PatternRule("\"", "\"", token, '\\', true, true, true); //$NON-NLS-1$ //$NON-NLS-2$
rules.add(stringRule); rules.add(stringRule);

115
core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/FastCPartitionScanner.java
View file

@ -1,5 +1,5 @@
/******************************************************************************* /*******************************************************************************
* Copyright (c) 2000, 2006 IBM Corporation and others. * Copyright (c) 2000, 2008 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials * All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0 * are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at * which accompanies this distribution, and is available at
@ -37,6 +37,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
private static final int STRING= 4; private static final int STRING= 4;
private static final int PREPROCESSOR= 5; private static final int PREPROCESSOR= 5;
private static final int PREPROCESSOR_MULTI_LINE_COMMENT= 6; private static final int PREPROCESSOR_MULTI_LINE_COMMENT= 6;
private static final int PREPROCESSOR_STRING= 7;
// beginning of prefixes and postfixes // beginning of prefixes and postfixes
private static final int NONE= 0; private static final int NONE= 0;
@ -46,6 +47,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
private static final int STAR= 4; // postfix for MULTI_LINE_COMMENT private static final int STAR= 4; // postfix for MULTI_LINE_COMMENT
private static final int CARRIAGE_RETURN=5; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT private static final int CARRIAGE_RETURN=5; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
private static final int BACKSLASH_CR= 6; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT private static final int BACKSLASH_CR= 6; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
private static final int BACKSLASH_BACKSLASH= 7; // postfix for STRING, CHARACTER
/** The scanner. */ /** The scanner. */
private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000); // faster implementation private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000); // faster implementation
@ -65,7 +67,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
private boolean fFirstCharOnLine= true; private boolean fFirstCharOnLine= true;
// emulate CPartitionScanner // emulate CPartitionScanner
private boolean fEmulate= false; private final boolean fEmulate;
private int fCCodeOffset; private int fCCodeOffset;
private int fCCodeLength; private int fCCodeLength;
@ -76,7 +78,8 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
new Token(C_CHARACTER), new Token(C_CHARACTER),
new Token(C_STRING), new Token(C_STRING),
new Token(C_PREPROCESSOR), new Token(C_PREPROCESSOR),
new Token(C_PREPROCESSOR) new Token(C_MULTI_LINE_COMMENT),
new Token(C_PREPROCESSOR),
}; };
public FastCPartitionScanner(boolean emulate) { public FastCPartitionScanner(boolean emulate) {
@ -128,7 +131,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
case '\r': case '\r':
fFirstCharOnLine= true; fFirstCharOnLine= true;
if (!fEmulate && fLast == BACKSLASH) { if (!fEmulate && fLast == BACKSLASH || fLast == BACKSLASH_BACKSLASH) {
fLast= BACKSLASH_CR; fLast= BACKSLASH_CR;
fTokenLength++; fTokenLength++;
continue; continue;
@ -171,11 +174,18 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
} }
case '\\': case '\\':
if (fLast == BACKSLASH) { switch (fState) {
consume(); case CHARACTER:
continue; case STRING:
case PREPROCESSOR_STRING:
fTokenLength++;
fLast= fLast == BACKSLASH ? BACKSLASH_BACKSLASH : BACKSLASH;
continue;
default:
fTokenLength++;
fLast= BACKSLASH;
continue;
} }
break;
case '\n': case '\n':
fFirstCharOnLine= true; fFirstCharOnLine= true;
@ -184,9 +194,10 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
case CHARACTER: case CHARACTER:
case STRING: case STRING:
case PREPROCESSOR: case PREPROCESSOR:
case PREPROCESSOR_STRING:
// assert(fTokenLength > 0); // assert(fTokenLength > 0);
// if last char was a backslash then we have an escaped line // if last char was a backslash then we have spliced line
if (fLast != BACKSLASH && fLast != BACKSLASH_CR) { if (fLast != BACKSLASH && fLast != BACKSLASH_CR && fLast != BACKSLASH_BACKSLASH) {
return postFix(fState); return postFix(fState);
} }
@ -202,6 +213,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
case CHARACTER: case CHARACTER:
case STRING: case STRING:
case PREPROCESSOR: case PREPROCESSOR:
case PREPROCESSOR_STRING:
int last; int last;
int newState; int newState;
@ -352,51 +364,67 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
break; break;
case SINGLE_LINE_COMMENT: case SINGLE_LINE_COMMENT:
switch (ch) { consume();
case '\\': break;
fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
fTokenLength++;
break;
default:
consume();
break;
}
break;
case PREPROCESSOR: case PREPROCESSOR:
switch (ch) { switch (ch) {
case '\\':
fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
fTokenLength++;
break;
case '/': case '/':
if (fLast == SLASH) { if (fLast == SLASH) {
consume(); if (fTokenLength - getLastLength(fLast) > 0) {
break; return preFix(fState, SINGLE_LINE_COMMENT, SLASH, 2);
} else {
preFix(fState, SINGLE_LINE_COMMENT, SLASH, 2);
fTokenOffset += fTokenLength;
fTokenLength= fPrefixLength;
break;
}
} else { } else {
fTokenLength++; consume();
fLast= SLASH; fLast= SLASH;
break;
} }
break;
case '*': case '*':
if (fLast == SLASH) { if (fLast == SLASH) {
fState= PREPROCESSOR_MULTI_LINE_COMMENT; if (fTokenLength - getLastLength(fLast) > 0) {
consume(); return preFix(fState, PREPROCESSOR_MULTI_LINE_COMMENT, SLASH_STAR, 2);
break; } else {
} else { preFix(fState, PREPROCESSOR_MULTI_LINE_COMMENT, SLASH_STAR, 2);
consume(); fTokenOffset += fTokenLength;
break; fTokenLength= fPrefixLength;
break;
}
} }
case '"':
if (fLast != BACKSLASH) {
fState= PREPROCESSOR_STRING;
}
consume();
break;
default: default:
consume(); consume();
break; break;
} }
break; break;
case PREPROCESSOR_STRING:
switch (ch) {
case '"':
if (fLast != BACKSLASH) {
fState= PREPROCESSOR;
}
consume();
break;
default:
consume();
break;
}
break;
case PREPROCESSOR_MULTI_LINE_COMMENT: case PREPROCESSOR_MULTI_LINE_COMMENT:
switch (ch) { switch (ch) {
case '*': case '*':
@ -406,7 +434,9 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
case '/': case '/':
if (fLast == STAR) { if (fLast == STAR) {
IToken token= postFix(fState);
fState= PREPROCESSOR; fState= PREPROCESSOR;
return token;
} }
consume(); consume();
break; break;
@ -434,17 +464,12 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
default: default:
consume(); consume();
break; break;
} }
break; break;
case STRING: case STRING:
switch (ch) { switch (ch) {
case '\\':
fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
fTokenLength++;
break;
case '\"': case '\"':
if (fLast != BACKSLASH) { if (fLast != BACKSLASH) {
return postFix(STRING); return postFix(STRING);
@ -462,11 +487,6 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
case CHARACTER: case CHARACTER:
switch (ch) { switch (ch) {
case '\\':
fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
fTokenLength++;
break;
case '\'': case '\'':
if (fLast != BACKSLASH) { if (fLast != BACKSLASH) {
return postFix(CHARACTER); return postFix(CHARACTER);
@ -501,6 +521,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
case SLASH_STAR: case SLASH_STAR:
case BACKSLASH_CR: case BACKSLASH_CR:
case BACKSLASH_BACKSLASH:
return 2; return 2;
} }

45
core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/asm/AsmPartitionScanner.java
View file

@ -1,5 +1,5 @@
/******************************************************************************* /*******************************************************************************
* Copyright (c) 2000, 2007 IBM Corporation and others. * Copyright (c) 2000, 2008 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials * All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0 * are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at * which accompanies this distribution, and is available at
@ -71,6 +71,7 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
private static final int STAR= 4; // postfix for MULTI_LINE_COMMENT private static final int STAR= 4; // postfix for MULTI_LINE_COMMENT
private static final int CARRIAGE_RETURN=5; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT private static final int CARRIAGE_RETURN=5; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
private static final int BACKSLASH_CR= 6; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT private static final int BACKSLASH_CR= 6; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
private static final int BACKSLASH_BACKSLASH= 7; // postfix for STRING, CHARACTER
/** The scanner. */ /** The scanner. */
private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000); private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000);
@ -185,7 +186,7 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
case '\r': case '\r':
fFirstCharOnLine= true; fFirstCharOnLine= true;
if (fLast == BACKSLASH) { if (fLast == BACKSLASH || fLast == BACKSLASH_BACKSLASH) {
fLast= BACKSLASH_CR; fLast= BACKSLASH_CR;
fTokenLength++; fTokenLength++;
continue; continue;
@ -221,11 +222,17 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
} }
case '\\': case '\\':
if (fLast == BACKSLASH) { switch (fState) {
consume(); case CHARACTER:
continue; case STRING:
fTokenLength++;
fLast= fLast == BACKSLASH ? BACKSLASH_BACKSLASH : BACKSLASH;
continue;
default:
fTokenLength++;
fLast= BACKSLASH;
continue;
} }
break;
case '\n': case '\n':
fFirstCharOnLine= true; fFirstCharOnLine= true;
@ -236,7 +243,7 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
case PREPROCESSOR: case PREPROCESSOR:
// assert(fTokenLength > 0); // assert(fTokenLength > 0);
// if last char was a backslash then we have an escaped line // if last char was a backslash then we have an escaped line
if (fLast != BACKSLASH && fLast != BACKSLASH_CR) { if (fLast != BACKSLASH && fLast != BACKSLASH_CR && fLast != BACKSLASH_BACKSLASH) {
return postFix(fState); return postFix(fState);
} }
@ -431,25 +438,11 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
break; break;
case SINGLE_LINE_COMMENT: case SINGLE_LINE_COMMENT:
switch (ch) { consume();
case '\\': break;
fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
fTokenLength++;
break;
default:
consume();
break;
}
break;
case PREPROCESSOR: case PREPROCESSOR:
switch (ch) { switch (ch) {
case '\\':
fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
fTokenLength++;
break;
case '/': case '/':
if (fLast == SLASH) { if (fLast == SLASH) {
consume(); consume();
@ -525,11 +518,6 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
case STRING: case STRING:
switch (ch) { switch (ch) {
case '\\':
fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
fTokenLength++;
break;
case '\"': case '\"':
if (fLast != BACKSLASH) { if (fLast != BACKSLASH) {
return postFix(STRING); return postFix(STRING);
@ -579,6 +567,7 @@ public final class AsmPartitionScanner implements IPartitionTokenScanner, ICPart
case SLASH_STAR: case SLASH_STAR:
case BACKSLASH_CR: case BACKSLASH_CR:
case BACKSLASH_BACKSLASH:
return 2; return 2;
} }