diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java index 34151318331..08c321be868 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/LexerTests.java @@ -142,7 +142,7 @@ public class LexerTests extends BaseTestCase { token(IToken.tBITOR); token(IToken.tLBRACE); token(IToken.tRBRACE); - token(IToken.tCOMPL); + token(IToken.tBITCOMPLEMENT); eof(); init("a??/\nb"); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java index 82482ebb3da..792ecdf66cd 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java @@ -43,7 +43,6 @@ final public class Lexer { public static final int tSYSTEM_HEADER_NAME = IToken.FIRST_RESERVED_SCANNER + 4; private static final int END_OF_INPUT = -1; - private static final int LINE_SPLICE_SEQUENCE = -2; private static final int ORIGIN_LEXER = OffsetLimitReachedException.ORIGIN_LEXER; public final static class LexerOptions implements Cloneable { @@ -210,6 +209,7 @@ final public class Lexer { * @throws OffsetLimitReachedException when completion is requested in a literal or an header-name. */ public Token nextDirective() throws OffsetLimitReachedException { + fInsideIncludeDirective= false; final Token t= fToken; boolean haveNL= t==null || t.getType() == tNEWLINE; while(true) { @@ -217,14 +217,38 @@ final public class Lexer { haveNL= false; final int start= fOffset; final int c= fCharPhase3; - final int d= nextCharPhase3(); + + // optimization avoids calling nextCharPhase3 + int d; + final int pos= fEndOffset; + if (pos+1 >= fLimit) { + d= nextCharPhase3(); + } + else { + d= fInput[pos]; + switch(d) { + case '\\': + d= nextCharPhase3(); + break; + case '?': + if (fInput[pos+1] == '?') { + d= nextCharPhase3(); + break; + } + // no break; + default: + fOffset= pos; + fCharPhase3= d; + fEndOffset= pos+1; + } + } + switch(c) { case END_OF_INPUT: fToken= newToken(Lexer.tEND_OF_INPUT, start); return fToken; case '\n': haveNL= true; - fInsideIncludeDirective= false; continue; case ' ': case '\t': @@ -261,6 +285,7 @@ final public class Lexer { continue; } if (hadNL) { + fFirstTokenAfterNewline= true; fToken= newToken(IToken.tPOUND, start); return fToken; } @@ -637,25 +662,21 @@ final public class Lexer { } private void blockComment(final int start) { - int c= nextCharPhase3(); - while(true) { - switch (c) { - case END_OF_INPUT: - fLog.handleComment(true, start, fOffset); - return; - case '*': - c= nextCharPhase3(); - if (c == '/') { + // we can ignore line-splices, trigraphs and windows newlines when searching for the '*' + int pos= fEndOffset; + while(pos < fLimit) { + if (fInput[pos++] == '*') { + fEndOffset= pos; + if (nextCharPhase3() == '/') { nextCharPhase3(); fLog.handleComment(true, start, fOffset); return; } - break; - default: - c= nextCharPhase3(); - break; } } + fCharPhase3= END_OF_INPUT; + fOffset= fEndOffset= pos; + fLog.handleComment(true, start, pos); } private void lineComment(final int start) { @@ -907,75 +928,72 @@ final public class Lexer { /** * Perform phase 1-3: Replace \r\n with \n, handle trigraphs, detect line-splicing. - * Changes fOffset, fEndOffset and fCharPhase3. + * Changes fOffset, fEndOffset and fCharPhase3, stateless otherwise. */ private int nextCharPhase3() { - int offset; - int c; + int pos= fEndOffset; do { - offset= fEndOffset; - c= fetchCharPhase3(offset); // changes fEndOffset - } - while(c == LINE_SPLICE_SEQUENCE); - - fOffset= offset; - fCharPhase3= c; - return c; - } - - /** - * Perform phase 1-3: Replace \r\n with \n, handle trigraphs, detect line-splicing. - * Changes fEndOffset, but is stateless otherwise. - */ - private int fetchCharPhase3(int pos) { - if (pos >= fLimit) { - fEndOffset= fLimit; - return END_OF_INPUT; - } - final char c= fInput[pos++]; - switch(c) { + if (pos+1 >= fLimit) { + if (pos >= fLimit) { + fOffset= fLimit; + fEndOffset= fLimit; + fCharPhase3= END_OF_INPUT; + return END_OF_INPUT; + } + fOffset= pos; + fEndOffset= pos+1; + fCharPhase3= fInput[pos]; + return fCharPhase3; + } + + final char c= fInput[pos]; + fOffset= pos; + fEndOffset= ++pos; + fCharPhase3= c; + switch(c) { // windows line-ending case '\r': - if (pos < fLimit && fInput[pos] == '\n') { - fEndOffset= pos+1; - return '\n'; - } - fEndOffset= pos; - return c; + if (fInput[pos] == '\n') { + fEndOffset= pos+1; + fCharPhase3= '\n'; + return '\n'; + } + return c; - // trigraph sequences - case '?': - if (pos+1 >= fLimit || fInput[pos] != '?') { + // trigraph sequences + case '?': + if (fInput[pos] != '?' || pos+1 >= fLimit) { + return c; + } + final char trigraph= checkTrigraph(fInput[pos+1]); + if (trigraph == 0) { + return c; + } + if (trigraph != '\\') { + fEndOffset= pos+2; + fCharPhase3= trigraph; + return trigraph; + } + pos+= 2; + // no break, handle backslash + + case '\\': + final int lsPos= findEndOfLineSpliceSequence(pos); + if (lsPos > pos) { + pos= lsPos; + continue; + } fEndOffset= pos; + fCharPhase3= '\\'; + return '\\'; // don't return c, it may be a '?' + + default: return c; } - final char trigraph= checkTrigraph(fInput[pos+1]); - if (trigraph == 0) { - fEndOffset= pos; - return c; - } - if (trigraph != '\\') { - fEndOffset= pos+2; - return trigraph; - } - pos+= 2; - // no break, handle backslash - - case '\\': - final int lsPos= findEndOfLineSpliceSequence(pos); - if (lsPos > pos) { - fEndOffset= lsPos; - return LINE_SPLICE_SEQUENCE; - } - fEndOffset= pos; - return '\\'; // don't return c, it may be a '?' - - default: - fEndOffset= pos; - return c; } + while(true); } - + /** * Maps a trigraph to the character it encodes. * @param c trigraph without leading question marks. @@ -1059,12 +1077,8 @@ final public class Lexer { final char[] result= new char[imageLength]; markPhase3(); fEndOffset= offset; - int idx= 0; - while (idx