mirrors/cdt
1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-06-09 10:46:02 +02:00
Code Issues Wiki Activity

C99 preprocessor is now generic and can be used with any lexer

Browse source
This commit is contained in:
Mike Kucera 2007-10-15 19:50:35 +00:00
parent 7cfc4446c3
commit 61f44ef073
5 changed files with 136 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/core/dom/upc/UPCLanguage.java
View file

@ -11,6 +11,7 @@
package org.eclipse.cdt.core.dom.upc; package org.eclipse.cdt.core.dom.upc;
import org.eclipse.cdt.core.dom.c99.BaseExtensibleLanguage; import org.eclipse.cdt.core.dom.c99.BaseExtensibleLanguage;
import org.eclipse.cdt.core.dom.c99.IC99TokenCollector;
import org.eclipse.cdt.core.dom.c99.IKeywordMap; import org.eclipse.cdt.core.dom.c99.IKeywordMap;
import org.eclipse.cdt.core.dom.c99.ILexerFactory; import org.eclipse.cdt.core.dom.c99.ILexerFactory;
import org.eclipse.cdt.core.dom.c99.IPPTokenComparator; import org.eclipse.cdt.core.dom.c99.IPPTokenComparator;
@ -21,6 +22,7 @@ import org.eclipse.cdt.core.dom.parser.upc.UPCKeywordMap;
import org.eclipse.cdt.internal.core.dom.parser.upc.UPCLexerFactory; import org.eclipse.cdt.internal.core.dom.parser.upc.UPCLexerFactory;
import org.eclipse.cdt.internal.core.dom.parser.upc.UPCPPTokenComparator; import org.eclipse.cdt.internal.core.dom.parser.upc.UPCPPTokenComparator;
import org.eclipse.cdt.internal.core.dom.parser.upc.UPCParser; import org.eclipse.cdt.internal.core.dom.parser.upc.UPCParser;
import org.eclipse.cdt.internal.core.dom.parser.upc.UPCTokenCollector;
/** /**
@ -76,4 +78,8 @@ public class UPCLanguage extends BaseExtensibleLanguage {
protected IPPTokenComparator getTokenComparator() { protected IPPTokenComparator getTokenComparator() {
return new UPCPPTokenComparator(); return new UPCPPTokenComparator();
} }
protected IC99TokenCollector getTokenCollector() {
return new UPCTokenCollector();
}
} }

96
upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/internal/core/dom/parser/upc/UPCPPTokenComparator.java
View file

@ -11,16 +11,54 @@
package org.eclipse.cdt.internal.core.dom.parser.upc; package org.eclipse.cdt.internal.core.dom.parser.upc;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.*; import static org.eclipse.cdt.core.dom.parser.c99.PPToken.AND;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.ANDAND;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.BANG;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.CARET;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.CHARCONST;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.COLON;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.COMMA;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.DOTDOTDOT;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.EOF;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.EQ;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.GE;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.GT;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.HASH;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.HASHHASH;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.IDENT;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.INTEGER;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.LE;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.LEFTSHIFT;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.LPAREN;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.LT;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.MINUS;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.MULTI_LINE_COMMENT;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.NE;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.NEWLINE;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.OR;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.OROR;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.PERCENT;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.PLACEMARKER;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.PLUS;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.QUESTION;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.RIGHTSHIFT;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.RPAREN;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.SINGLE_LINE_COMMENT;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.SLASH;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.STAR;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.STRINGLIT;
import static org.eclipse.cdt.core.dom.parser.c99.PPToken.TILDE;
import lpg.lpgjavaruntime.IToken;
import org.eclipse.cdt.core.dom.c99.IPPTokenComparator; import org.eclipse.cdt.core.dom.c99.IPPTokenComparator;
import org.eclipse.cdt.core.dom.parser.c99.IToken;
import org.eclipse.cdt.core.dom.parser.c99.PPToken; import org.eclipse.cdt.core.dom.parser.c99.PPToken;
import org.eclipse.cdt.internal.core.dom.parser.c99.C99Parsersym;
import org.eclipse.cdt.internal.core.dom.parser.c99.preprocessor.SynthesizedToken; import org.eclipse.cdt.internal.core.dom.parser.c99.preprocessor.SynthesizedToken;
import org.eclipse.cdt.internal.core.dom.parser.c99.preprocessor.Token; import org.eclipse.cdt.internal.core.dom.parser.c99.preprocessor.Token;
public class UPCPPTokenComparator implements IPPTokenComparator { public class UPCPPTokenComparator implements IPPTokenComparator<IToken> {
private static final int PLACEMARKER_VALUE = Integer.MAX_VALUE;
public PPToken getKind(IToken token) { public PPToken getKind(IToken token) {
if(token == null) if(token == null)
@ -37,11 +75,37 @@ public class UPCPPTokenComparator implements IPPTokenComparator {
case UPCParsersym.TK_EOF_TOKEN : return EOF; case UPCParsersym.TK_EOF_TOKEN : return EOF;
case UPCParsersym.TK_stringlit : return STRINGLIT; case UPCParsersym.TK_stringlit : return STRINGLIT;
case UPCParsersym.TK_integer : return INTEGER; case UPCParsersym.TK_integer : return INTEGER;
case UPCParsersym.TK_LT : return LEFT_ANGLE_BRACKET;
case UPCParsersym.TK_GT : return RIGHT_ANGLE_BRACKET;
case UPCParsersym.TK_SingleLineComment : return SINGLE_LINE_COMMENT; case UPCParsersym.TK_SingleLineComment : return SINGLE_LINE_COMMENT;
case UPCParsersym.TK_MultiLineComment : return MULTI_LINE_COMMENT; case UPCParsersym.TK_MultiLineComment : return MULTI_LINE_COMMENT;
case UPCParsersym.TK_identifier : return IDENT; case UPCParsersym.TK_identifier : return IDENT;
case UPCParsersym.TK_charconst : return CHARCONST;
case UPCParsersym.TK_And : return AND;
case UPCParsersym.TK_Star : return STAR;
case UPCParsersym.TK_Plus : return PLUS;
case UPCParsersym.TK_Minus : return MINUS;
case UPCParsersym.TK_Tilde : return TILDE;
case UPCParsersym.TK_Bang : return BANG;
case UPCParsersym.TK_Slash : return SLASH;
case UPCParsersym.TK_Percent : return PERCENT;
case UPCParsersym.TK_RightShift : return RIGHTSHIFT;
case UPCParsersym.TK_LeftShift : return LEFTSHIFT;
case UPCParsersym.TK_LT : return LT;
case UPCParsersym.TK_GT : return GT;
case UPCParsersym.TK_LE : return LE;
case UPCParsersym.TK_GE : return GE;
case UPCParsersym.TK_EQ : return EQ;
case UPCParsersym.TK_NE : return NE;
case UPCParsersym.TK_Caret : return CARET;
case UPCParsersym.TK_Or : return OR;
case UPCParsersym.TK_AndAnd : return ANDAND;
case UPCParsersym.TK_OrOr : return OROR;
case UPCParsersym.TK_Question : return QUESTION;
case UPCParsersym.TK_Colon : return COLON;
// TODO: will removing this case cause the switch to compile into a tableswitch bytecode?
// tableswitch is faster than lookupswitch
case PLACEMARKER_VALUE : return PLACEMARKER;
} }
return null; return null;
} }
@ -51,12 +115,12 @@ public class UPCPPTokenComparator implements IPPTokenComparator {
int kind; int kind;
switch(tokenToMake) { switch(tokenToMake) {
case KIND_IDENTIFIER : kind = UPCParsersym.TK_identifier; break; case KIND_IDENTIFIER : kind = UPCParsersym.TK_identifier; break;
case KIND_EOF : kind = UPCParsersym.TK_EOF_TOKEN; break;
case KIND_COMPLETION : kind = UPCParsersym.TK_Completion; break; case KIND_COMPLETION : kind = UPCParsersym.TK_Completion; break;
case KIND_END_OF_COMPLETION : kind = UPCParsersym.TK_EndOfCompletion; break; case KIND_END_OF_COMPLETION : kind = UPCParsersym.TK_EndOfCompletion; break;
case KIND_INTEGER : kind = UPCParsersym.TK_integer; break; case KIND_INTEGER : kind = UPCParsersym.TK_integer; break;
case KIND_STRINGLIT : kind = UPCParsersym.TK_stringlit; break; case KIND_STRINGLIT : kind = UPCParsersym.TK_stringlit; break;
case KIND_INVALID : kind = UPCParsersym.TK_Invalid; break; case KIND_INVALID : kind = UPCParsersym.TK_Invalid; break;
case KIND_PLACEMARKER : kind = PLACEMARKER_VALUE; break;
default : kind = UPCParsersym.TK_Invalid; break; default : kind = UPCParsersym.TK_Invalid; break;
} }
@ -65,13 +129,25 @@ public class UPCPPTokenComparator implements IPPTokenComparator {
public IToken cloneToken(IToken token) { public IToken cloneToken(IToken token) {
if(token instanceof Token) { if(token instanceof Token) {
return ((Token)token).clone(); return (IToken)((Token)token).clone();
} }
throw new RuntimeException("don't know what kind of token that is"); //$NON-NLS-1$ throw new RuntimeException("don't know what kind of token that is"); //$NON-NLS-1$
} }
public String[] getLPGOrderedTerminalSymbols() {
return UPCParsersym.orderedTerminalSymbols; public int getEndOffset(IToken token) {
return token.getEndOffset();
} }
public int getStartOffset(IToken token) {
return token.getStartOffset();
}
public void setEndOffset(IToken token, int offset) {
token.setEndOffset(offset);
}
public void setStartOffset(IToken token, int offset) {
token.setStartOffset(offset);
}
} }

1
upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/internal/core/dom/parser/upc/UPCParser.g
View file

@ -30,6 +30,7 @@ import org.eclipse.cdt.internal.core.dom.parser.c99.C99Lexer;
import org.eclipse.cdt.internal.core.dom.parser.c99.C99Parsersym; import org.eclipse.cdt.internal.core.dom.parser.c99.C99Parsersym;
import org.eclipse.cdt.core.dom.parser.upc.UPCKeywordMap; import org.eclipse.cdt.core.dom.parser.upc.UPCKeywordMap;
import org.eclipse.cdt.core.dom.parser.upc.UPCParserAction; import org.eclipse.cdt.core.dom.parser.upc.UPCParserAction;
import org.eclipse.cdt.core.dom.c99.IPreprocessorTokenCollector;
./ ./
$End $End

3
upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/internal/core/dom/parser/upc/UPCParser.java
View file

@ -21,6 +21,7 @@ import org.eclipse.cdt.core.dom.ast.*;
import org.eclipse.cdt.core.dom.c99.IParserActionTokenProvider; import org.eclipse.cdt.core.dom.c99.IParserActionTokenProvider;
import org.eclipse.cdt.core.dom.c99.IParser; import org.eclipse.cdt.core.dom.c99.IParser;
import org.eclipse.cdt.core.dom.c99.IParseResult; import org.eclipse.cdt.core.dom.c99.IParseResult;
import org.eclipse.cdt.core.dom.c99.IPreprocessorTokenCollector;
import org.eclipse.cdt.core.dom.parser.c99.C99ParseResult; import org.eclipse.cdt.core.dom.parser.c99.C99ParseResult;
import org.eclipse.cdt.core.dom.parser.c99.C99ParserAction; import org.eclipse.cdt.core.dom.parser.c99.C99ParserAction;
import org.eclipse.cdt.core.dom.ast.IASTCompletionNode; import org.eclipse.cdt.core.dom.ast.IASTCompletionNode;
@ -35,7 +36,7 @@ import org.eclipse.cdt.internal.core.dom.parser.c99.C99Parsersym;
import org.eclipse.cdt.core.dom.parser.upc.UPCKeywordMap; import org.eclipse.cdt.core.dom.parser.upc.UPCKeywordMap;
import org.eclipse.cdt.core.dom.parser.upc.UPCParserAction; import org.eclipse.cdt.core.dom.parser.upc.UPCParserAction;
public class UPCParser extends PrsStream implements RuleAction , IParserActionTokenProvider, IParser public class UPCParser extends PrsStream implements RuleAction , IParserActionTokenProvider, IParser, IPreprocessorTokenCollector<IToken>
{ {
private static ParseTable prs = new UPCParserprs(); private static ParseTable prs = new UPCParserprs();
private BacktrackingParser btParser; private BacktrackingParser btParser;

39
upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/internal/core/dom/parser/upc/UPCTokenCollector.java Normal file
View file

@ -0,0 +1,39 @@
package org.eclipse.cdt.internal.core.dom.parser.upc;
import lpg.lpgjavaruntime.IToken;
import org.eclipse.cdt.core.dom.c99.IC99TokenCollector;
import org.eclipse.cdt.core.dom.c99.IParser;
import org.eclipse.cdt.core.dom.c99.IPreprocessorTokenCollector;
import org.eclipse.cdt.internal.core.dom.parser.c99.C99Parser;
import org.eclipse.cdt.internal.core.dom.parser.c99.preprocessor.SynthesizedToken;
import org.eclipse.cdt.internal.core.dom.parser.c99.preprocessor.Token;
/**
* An LPG token stream must start with a dummy token and end with an EOF token.
* @author Mike
*
*/
public class UPCTokenCollector implements IC99TokenCollector {
private IPreprocessorTokenCollector<IToken> parser;
public void setParser(IParser parser) {
this.parser = (IPreprocessorTokenCollector<IToken>) parser; // Total HACK!
this.parser.addToken(Token.DUMMY_TOKEN);
}
public void addCommentToken(IToken token) {
parser.addCommentToken(token);
}
public void addToken(IToken token) {
parser.addToken(token);
}
public void done(int translationUnitSize) {
parser.addToken(new SynthesizedToken(translationUnitSize, translationUnitSize, UPCParsersym.TK_EOF_TOKEN, ""));
}
}