From 4466a9e8193b8bcf2febf4f7fad18566aec0804e Mon Sep 17 00:00:00 2001 From: Markus Schorn Date: Tue, 30 Oct 2007 18:36:11 +0000 Subject: [PATCH] Performance Test for the preprocessor --- .../tests/scanner/PreprocessorSpeedTest.java | 194 ++++++++++++++++++ .../tests/scanner2/Scanner2SpeedTest.java | 13 +- .../parser/tests/scanner2/SpeedTest2.java | 14 +- .../core/parser/scanner/CPreprocessor.java | 2 +- .../internal/core/parser/scanner/Lexer.java | 76 +++++-- 5 files changed, 265 insertions(+), 34 deletions(-) create mode 100644 core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorSpeedTest.java diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorSpeedTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorSpeedTest.java new file mode 100644 index 00000000000..29e9c8df562 --- /dev/null +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorSpeedTest.java @@ -0,0 +1,194 @@ +/******************************************************************************* + * Copyright (c) 2005, 2007 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM Corporation - initial API and implementation + * Markus Schorn (Wind River Systems) + *******************************************************************************/ +package org.eclipse.cdt.core.parser.tests.scanner; + +import java.io.FileOutputStream; +import java.io.PrintStream; +import java.util.Hashtable; +import java.util.Map; + +import org.eclipse.cdt.core.dom.ICodeReaderFactory; +import org.eclipse.cdt.core.dom.parser.IScannerExtensionConfiguration; +import org.eclipse.cdt.core.dom.parser.c.GCCScannerExtensionConfiguration; +import org.eclipse.cdt.core.dom.parser.cpp.GPPScannerExtensionConfiguration; +import org.eclipse.cdt.core.parser.CodeReader; +import org.eclipse.cdt.core.parser.EndOfFileException; +import org.eclipse.cdt.core.parser.IScannerInfo; +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.NullLogService; +import org.eclipse.cdt.core.parser.ParserLanguage; +import org.eclipse.cdt.core.parser.ParserMode; +import org.eclipse.cdt.core.parser.ScannerInfo; +import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTTranslationUnit; +import org.eclipse.cdt.internal.core.parser.scanner.CPreprocessor; +import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory; + +public class PreprocessorSpeedTest { + + private PrintStream stream; + + public static void main(String[] args) { + try { + PrintStream stream = null; + if (args.length > 0) + stream = new PrintStream(new FileOutputStream(args[0])); + + new PreprocessorSpeedTest().runTest(stream, 30); + } catch (Exception e) { + System.out.println(e); + } + } + + public void test() throws Exception { + runTest(10); + } + + public void runTest(PrintStream stream, int n) throws Exception { + this.stream = stream; + runTest(n); + } + + private void runTest(int n) throws Exception { + String code = + "#include \n" + + "#include \n" + + "#include \n"; + + CodeReader reader = new CodeReader(code.toCharArray()); + IScannerInfo info = msvcScannerInfo(); + long totalTime = 0; + for (int i = 0; i < n; ++i) { + long time = testScan(reader, false, info, ParserLanguage.CPP); + if (i > 0) + totalTime += time; + } + + if (n > 1) { + System.out.println("Average Time: " + (totalTime / (n - 1)) + " millisecs"); + } + } + + protected long testScan(CodeReader reader, boolean quick, IScannerInfo info, ParserLanguage lang) throws Exception { + ICodeReaderFactory readerFactory= FileCodeReaderFactory.getInstance(); + IScannerExtensionConfiguration scannerConfig; + if (lang == ParserLanguage.C) { + scannerConfig= new GCCScannerExtensionConfiguration(); + } + else { + scannerConfig= new GPPScannerExtensionConfiguration(); + } + ParserMode mode = ParserMode.COMPLETE_PARSE; + CPreprocessor cpp= new CPreprocessor(reader, info, lang, new NullLogService(), scannerConfig, readerFactory); + cpp.getLocationMap().setRootNode(new CPPASTTranslationUnit()); + long startTime = System.currentTimeMillis(); + int count = 0; + try { + while (true) { + IToken t = cpp.nextToken(); + + if (stream != null) + stream.println(t.getImage()); + + if (t == null) + break; + ++count; + + } + } catch (EndOfFileException e2) { + } + long totalTime = System.currentTimeMillis() - startTime; + System.out.println( "Resulting scan took " + totalTime + " millisecs " + + count + " tokens"); + return totalTime; + } + + protected IScannerInfo msvcScannerInfo() { + Map definitions = new Hashtable(); + String [] includePaths = new String[] { + "C:\\Program Files\\Microsoft Visual Studio\\VC98\\Include" + }; + return new ScannerInfo( definitions, includePaths ); + } + + protected IScannerInfo mingwScannerInfo() { + // TODO It would be easier and more flexible if we used discovery for this + Map definitions = new Hashtable(); + definitions.put("__GNUC__", "3"); + definitions.put("__GNUC_MINOR__", "2"); + definitions.put("__GNUC_PATCHLEVEL__", "3"); + definitions.put("__GXX_ABI_VERSION", "102"); + definitions.put("_WIN32", ""); + definitions.put("__WIN32", ""); + definitions.put("__WIN32__", ""); + definitions.put("WIN32", ""); + definitions.put("__MINGW32__", ""); + definitions.put("__MSVCRT__", ""); + definitions.put("WINNT", ""); + definitions.put("_X86_", "1"); + definitions.put("__WINNT", ""); + definitions.put("_NO_INLINE__", ""); + definitions.put("__STDC_HOSTED__", "1"); + definitions.put("i386", ""); + definitions.put("__i386", ""); + definitions.put("__i386__", ""); + definitions.put("__tune_i586__", ""); + definitions.put("__tune_pentium__", ""); + definitions.put("__stdcall", "__attribute__((__stdcall__))"); + definitions.put("__cdecl", "__attribute__((__cdecl__))"); + definitions.put("__fastcall", "__attribute__((__fastcall__))"); + definitions.put("_stdcall", "__attribute__((__stdcall__))"); + definitions.put("_cdecl", "__attribute__((__cdecl__))"); + definitions.put("_fastcall", "__attribute__((__fastcall__))"); + definitions.put("__declspec(x)", "__attribute__((x))"); + definitions.put("__DEPRECATED", ""); + definitions.put("__EXCEPTIONS", ""); + + String [] includePaths = new String[] { + "c:/mingw/include/c++/3.2.3", + "c:/mingw/include/c++/3.2.3/mingw32", + "c:/mingw/include/c++/3.2.3/backward", + "c:/mingw/include", + "c:/mingw/lib/gcc-lib/mingw32/3.2.3/include" + }; + + return new ScannerInfo( definitions, includePaths ); + } + + protected IScannerInfo ydlScannerInfo() { + // TODO It would be easier and more flexible if we used discovery for this + Map definitions = new Hashtable(); + definitions.put("__GNUC__", "3"); + definitions.put("__GNUC_MINOR__", "3"); + definitions.put("__GNUC_PATCHLEVEL__", "3"); + definitions.put("_GNU_SOURCE", ""); + definitions.put("__unix__", ""); + definitions.put("__gnu_linux__", ""); + definitions.put("__linux__", ""); + definitions.put("unix", ""); + definitions.put("__unix", ""); + definitions.put("linux", ""); + definitions.put("__linux", ""); + definitions.put("__GNUG__", "3"); + + String [] includePaths = new String[] { + "/usr/include/g++", + "/usr/include/g++/powerpc-yellowdog-linux", + "/usr/include/g++/backward", + "/usr/local/include", + "/usr/lib/gcc-lib/powerpc-yellowdog-linux/3.3.3/include", + "/usr/include" + }; + + return new ScannerInfo( definitions, includePaths ); + } + +} diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2SpeedTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2SpeedTest.java index 4189f768de6..349c4275158 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2SpeedTest.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2SpeedTest.java @@ -16,7 +16,6 @@ */ package org.eclipse.cdt.core.parser.tests.scanner2; -import java.io.FileOutputStream; import java.io.PrintStream; import java.util.Collections; @@ -28,7 +27,7 @@ import org.eclipse.cdt.core.parser.IToken; import org.eclipse.cdt.core.parser.NullSourceElementRequestor; import org.eclipse.cdt.core.parser.ParserLanguage; import org.eclipse.cdt.core.parser.ParserMode; -import org.eclipse.cdt.internal.core.parser.scanner2.Scanner2; +import org.eclipse.cdt.internal.core.parser.scanner2.DOMScanner; /** * @author Doug Schaefer @@ -44,10 +43,10 @@ public class Scanner2SpeedTest extends SpeedTest2 { public static void main(String[] args) { try { PrintStream stream = null; - if (args.length > 0) - stream = new PrintStream(new FileOutputStream(args[0])); +// if (args.length > 0) +// stream = new PrintStream(new FileOutputStream("c:/tmp/dom.txt")); - new Scanner2SpeedTest().runTest(stream, 1); + new Scanner2SpeedTest().runTest(stream, 30); } catch (Exception e) { System.out.println(e); } @@ -69,7 +68,7 @@ public class Scanner2SpeedTest extends SpeedTest2 { "#include \n"; CodeReader reader = new CodeReader(code.toCharArray()); - IScannerInfo info = getScannerInfo(false); + IScannerInfo info = msvcScannerInfo(false); long totalTime = 0; for (int i = 0; i < n; ++i) { long time = testScan(reader, false, info, ParserLanguage.CPP); @@ -88,7 +87,7 @@ public class Scanner2SpeedTest extends SpeedTest2 { */ protected long testScan(CodeReader reader, boolean quick, IScannerInfo info, ParserLanguage lang) throws Exception { ParserMode mode = quick ? ParserMode.QUICK_PARSE : ParserMode.COMPLETE_PARSE; - Scanner2 scanner = createScanner(reader, info, mode, lang, CALLBACK, null, Collections.EMPTY_LIST ); + DOMScanner scanner = createScanner(reader, info, mode, lang, CALLBACK, null, Collections.EMPTY_LIST ); long startTime = System.currentTimeMillis(); int count = 0; try { diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/SpeedTest2.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/SpeedTest2.java index 42298d74e54..7ca8f33f279 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/SpeedTest2.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/SpeedTest2.java @@ -32,7 +32,8 @@ import org.eclipse.cdt.core.parser.ParserFactoryError; import org.eclipse.cdt.core.parser.ParserLanguage; import org.eclipse.cdt.core.parser.ParserMode; import org.eclipse.cdt.core.parser.ScannerInfo; -import org.eclipse.cdt.internal.core.parser.scanner2.Scanner2; +import org.eclipse.cdt.internal.core.parser.scanner2.DOMScanner; +import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory; // A test that just calculates the speed of the parser // Eventually, we'll peg a max time and fail the test if it exceeds it @@ -87,7 +88,7 @@ public class SpeedTest2 extends TestCase { return totalTime; } - public static Scanner2 createScanner( CodeReader code, IScannerInfo config, ParserMode mode, ParserLanguage language, ISourceElementRequestor requestor, IParserLogService log, List workingCopies ) throws ParserFactoryError + public static DOMScanner createScanner( CodeReader code, IScannerInfo config, ParserMode mode, ParserLanguage language, ISourceElementRequestor requestor, IParserLogService log, List workingCopies ) throws ParserFactoryError { if( config == null ) throw new ParserFactoryError( ParserFactoryError.Kind.NULL_CONFIG ); if( language == null ) throw new ParserFactoryError( ParserFactoryError.Kind.NULL_LANGUAGE ); @@ -99,7 +100,7 @@ public class SpeedTest2 extends TestCase { configuration = new GCCScannerExtensionConfiguration(); else configuration = new GPPScannerExtensionConfiguration(); - return new Scanner2( code, config, ourRequestor, ourMode, language, logService, workingCopies, configuration ); + return new DOMScanner( code, config, ourMode, language, logService, configuration, FileCodeReaderFactory.getInstance() ); } private static final ISourceElementRequestor CALLBACK = new NullSourceElementRequestor(); @@ -121,15 +122,16 @@ public class SpeedTest2 extends TestCase { return mingwScannerInfo(false); } - private IScannerInfo msvcScannerInfo(boolean quick) { + protected IScannerInfo msvcScannerInfo(boolean quick) { if( quick ) return new ScannerInfo(); Map definitions = new Hashtable(); //definitions.put( "__GNUC__", "3" ); //$NON-NLS-1$ //$NON-NLS-2$ String [] includePaths = new String[] { - "C:\\Program Files\\Microsoft SDK\\Include", - "C:\\Program Files\\Microsoft Visual C++ Toolkit 2003\\include" + "C:\\Program Files\\Microsoft Visual Studio\\VC98\\Include" +// "C:\\Program Files\\Microsoft Platform SDK\\Include", +// "C:\\Program Files\\Microsoft Visual C++ Toolkit 2003\\include" }; return new ScannerInfo( definitions, includePaths ); } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java index aa52535e84f..34bb9e620de 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java @@ -197,7 +197,7 @@ public class CPreprocessor implements ILexerLog, IScanner { IScannerExtensionConfiguration configuration, ICodeReaderFactory readerFactory) { fLanguage= language; fLog = log; - fCheckNumbers= true; // mstodo room for optimization. + fCheckNumbers= true; fAdditionalNumericLiteralSuffixes= nonNull(configuration.supportAdditionalNumericLiteralSuffixes()); fLexOptions.fSupportDollarInitializers= configuration.support$InIdentifiers(); fLexOptions.fSupportMinAndMax = configuration.supportMinAndMaxOperators(); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java index 29a9dc9b4c8..82482ebb3da 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java @@ -210,30 +210,66 @@ final public class Lexer { * @throws OffsetLimitReachedException when completion is requested in a literal or an header-name. */ public Token nextDirective() throws OffsetLimitReachedException { - Token t= fToken; + final Token t= fToken; boolean haveNL= t==null || t.getType() == tNEWLINE; - loop: while(true) { - t= fetchToken(); - if (haveNL) { - switch(t.getType()) { - case tEND_OF_INPUT: - case IToken.tPOUND: - break loop; + while(true) { + final boolean hadNL= haveNL; + haveNL= false; + final int start= fOffset; + final int c= fCharPhase3; + final int d= nextCharPhase3(); + switch(c) { + case END_OF_INPUT: + fToken= newToken(Lexer.tEND_OF_INPUT, start); + return fToken; + case '\n': + haveNL= true; + fInsideIncludeDirective= false; + continue; + case ' ': + case '\t': + case 0xb: // vertical tab + case '\f': + case '\r': + haveNL= hadNL; + continue; + + case '"': + stringLiteral(start, false); + continue; + + case '\'': + charLiteral(start, false); + continue; + + case '/': + switch (d) { + case '/': + nextCharPhase3(); + lineComment(start); + continue; + case '*': + nextCharPhase3(); + blockComment(start); + continue; } - haveNL= false; - } - else { - switch(t.getType()) { - case tEND_OF_INPUT: - break loop; - case tNEWLINE: - haveNL= true; - break; + continue; + + case '#': + if (d == '#') { + nextCharPhase3(); + continue; } + if (hadNL) { + fToken= newToken(IToken.tPOUND, start); + return fToken; + } + continue; + + default: + continue; } - } - fToken= t; - return t; + } } /**