Performance Test for the preprocessor

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorSpeedTest.java

@@ -0,0 +1,194 @@
+/*******************************************************************************
+ * Copyright (c) 2005, 2007 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     IBM Corporation - initial API and implementation
+ *     Markus Schorn (Wind River Systems)
+ *******************************************************************************/
+package org.eclipse.cdt.core.parser.tests.scanner;
+
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.Hashtable;
+import java.util.Map;
+
+import org.eclipse.cdt.core.dom.ICodeReaderFactory;
+import org.eclipse.cdt.core.dom.parser.IScannerExtensionConfiguration;
+import org.eclipse.cdt.core.dom.parser.c.GCCScannerExtensionConfiguration;
+import org.eclipse.cdt.core.dom.parser.cpp.GPPScannerExtensionConfiguration;
+import org.eclipse.cdt.core.parser.CodeReader;
+import org.eclipse.cdt.core.parser.EndOfFileException;
+import org.eclipse.cdt.core.parser.IScannerInfo;
+import org.eclipse.cdt.core.parser.IToken;
+import org.eclipse.cdt.core.parser.NullLogService;
+import org.eclipse.cdt.core.parser.ParserLanguage;
+import org.eclipse.cdt.core.parser.ParserMode;
+import org.eclipse.cdt.core.parser.ScannerInfo;
+import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTTranslationUnit;
+import org.eclipse.cdt.internal.core.parser.scanner.CPreprocessor;
+import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory;
+
+public class PreprocessorSpeedTest  {
+	
+	private PrintStream stream;
+
+	public static void main(String[] args) {
+		try {
+			PrintStream stream = null;
+			if (args.length > 0)
+				stream = new PrintStream(new FileOutputStream(args[0]));
+
+			new PreprocessorSpeedTest().runTest(stream, 30);
+		} catch (Exception e) {
+			System.out.println(e);
+		}
+	}
+
+	public void test() throws Exception {
+		runTest(10);
+	}
+
+	public void runTest(PrintStream stream, int n) throws Exception {
+		this.stream = stream;
+		runTest(n);
+	}
+	
+	private void runTest(int n) throws Exception {
+		String code = 
+			"#include <windows.h>\n" +
+			"#include <stdio.h>\n" +
+			"#include <iostream>\n";
+		
+		CodeReader reader = new CodeReader(code.toCharArray());
+		IScannerInfo info = msvcScannerInfo();
+		long totalTime = 0;
+		for (int i = 0; i < n; ++i) {
+			long time = testScan(reader, false, info, ParserLanguage.CPP);
+			if (i > 0)
+				totalTime += time;
+		}
+		
+		if (n > 1) {
+			System.out.println("Average Time: " + (totalTime / (n - 1)) + " millisecs");
+		}
+	}
+
+	protected long testScan(CodeReader reader, boolean quick, IScannerInfo info, ParserLanguage lang) throws Exception {
+		ICodeReaderFactory readerFactory= FileCodeReaderFactory.getInstance();
+		IScannerExtensionConfiguration scannerConfig;
+	    if (lang == ParserLanguage.C) {
+	    	scannerConfig= new GCCScannerExtensionConfiguration();
+	    }
+	    else {
+	    	scannerConfig= new GPPScannerExtensionConfiguration();
+	    }
+		ParserMode mode = ParserMode.COMPLETE_PARSE;
+		CPreprocessor cpp= new CPreprocessor(reader, info, lang, new NullLogService(), scannerConfig, readerFactory);
+		cpp.getLocationMap().setRootNode(new CPPASTTranslationUnit());
+		long startTime = System.currentTimeMillis();
+		int count = 0;
+		try {
+			while (true) {
+					IToken t = cpp.nextToken();
+					
+					if (stream != null)
+						stream.println(t.getImage());
+					
+					if (t == null)
+						break;
+					++count;
+				
+			}
+		} catch (EndOfFileException e2) {
+		}
+		long totalTime = System.currentTimeMillis() - startTime;
+		System.out.println( "Resulting scan took " + totalTime + " millisecs " +
+				count + " tokens");
+		return totalTime;
+	}
+	
+	protected IScannerInfo msvcScannerInfo() {
+		Map definitions = new Hashtable();
+		String [] includePaths = new String[] {
+			"C:\\Program Files\\Microsoft Visual Studio\\VC98\\Include"
+		};
+		return new ScannerInfo( definitions, includePaths );
+	}
+	
+	protected IScannerInfo mingwScannerInfo() {
+		// TODO It would be easier and more flexible if we used discovery for this
+		Map definitions = new Hashtable();
+		definitions.put("__GNUC__", "3");
+		definitions.put("__GNUC_MINOR__", "2");
+		definitions.put("__GNUC_PATCHLEVEL__", "3");
+		definitions.put("__GXX_ABI_VERSION", "102");
+		definitions.put("_WIN32", "");
+		definitions.put("__WIN32", "");
+		definitions.put("__WIN32__", "");
+		definitions.put("WIN32", "");
+		definitions.put("__MINGW32__", "");
+		definitions.put("__MSVCRT__", "");
+		definitions.put("WINNT", "");
+		definitions.put("_X86_", "1");
+		definitions.put("__WINNT", "");
+		definitions.put("_NO_INLINE__", "");
+		definitions.put("__STDC_HOSTED__", "1");
+		definitions.put("i386", "");
+		definitions.put("__i386", "");
+		definitions.put("__i386__", "");
+		definitions.put("__tune_i586__", "");
+		definitions.put("__tune_pentium__", "");
+		definitions.put("__stdcall", "__attribute__((__stdcall__))");
+		definitions.put("__cdecl", "__attribute__((__cdecl__))");
+		definitions.put("__fastcall", "__attribute__((__fastcall__))");
+		definitions.put("_stdcall", "__attribute__((__stdcall__))");
+		definitions.put("_cdecl", "__attribute__((__cdecl__))");
+		definitions.put("_fastcall", "__attribute__((__fastcall__))");
+		definitions.put("__declspec(x)", "__attribute__((x))");
+		definitions.put("__DEPRECATED", "");
+		definitions.put("__EXCEPTIONS", "");
+		
+		String [] includePaths = new String[] {
+			"c:/mingw/include/c++/3.2.3",
+			"c:/mingw/include/c++/3.2.3/mingw32",
+			"c:/mingw/include/c++/3.2.3/backward",
+			"c:/mingw/include",
+			"c:/mingw/lib/gcc-lib/mingw32/3.2.3/include"
+		};
+
+		return new ScannerInfo( definitions, includePaths );
+	}
+
+	protected IScannerInfo ydlScannerInfo() {
+		// TODO It would be easier and more flexible if we used discovery for this
+		Map definitions = new Hashtable();
+		definitions.put("__GNUC__", "3");
+		definitions.put("__GNUC_MINOR__", "3");
+		definitions.put("__GNUC_PATCHLEVEL__", "3");
+		definitions.put("_GNU_SOURCE", "");
+		definitions.put("__unix__", "");
+		definitions.put("__gnu_linux__", "");
+		definitions.put("__linux__", "");
+		definitions.put("unix", "");
+		definitions.put("__unix", "");
+		definitions.put("linux", "");
+		definitions.put("__linux", "");
+		definitions.put("__GNUG__", "3");
+		
+		String [] includePaths = new String[] {
+			"/usr/include/g++",
+			"/usr/include/g++/powerpc-yellowdog-linux",
+			"/usr/include/g++/backward",
+			"/usr/local/include",
+			"/usr/lib/gcc-lib/powerpc-yellowdog-linux/3.3.3/include",
+			"/usr/include"
+		};
+
+		return new ScannerInfo( definitions, includePaths );
+	}
+
+}

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2SpeedTest.java

@@ -16,7 +16,6 @@
  */
 package org.eclipse.cdt.core.parser.tests.scanner2;
 
-import java.io.FileOutputStream;
 import java.io.PrintStream;
 import java.util.Collections;
 
@@ -28,7 +27,7 @@ import org.eclipse.cdt.core.parser.IToken;
 import org.eclipse.cdt.core.parser.NullSourceElementRequestor;
 import org.eclipse.cdt.core.parser.ParserLanguage;
 import org.eclipse.cdt.core.parser.ParserMode;
-import org.eclipse.cdt.internal.core.parser.scanner2.Scanner2;
+import org.eclipse.cdt.internal.core.parser.scanner2.DOMScanner;
 
 /**
  * @author Doug Schaefer
@@ -44,10 +43,10 @@ public class Scanner2SpeedTest extends SpeedTest2 {
 	public static void main(String[] args) {
 		try {
 			PrintStream stream = null;
-			if (args.length > 0)
+//			if (args.length > 0)
-				stream = new PrintStream(new FileOutputStream(args[0]));
+//				stream = new PrintStream(new FileOutputStream("c:/tmp/dom.txt"));
 
-			new Scanner2SpeedTest().runTest(stream, 1);
+			new Scanner2SpeedTest().runTest(stream, 30);
 		} catch (Exception e) {
 			System.out.println(e);
 		}
@@ -69,7 +68,7 @@ public class Scanner2SpeedTest extends SpeedTest2 {
 			"#include <iostream>\n";
 		
 		CodeReader reader = new CodeReader(code.toCharArray());
-		IScannerInfo info = getScannerInfo(false);
+		IScannerInfo info = msvcScannerInfo(false);
 		long totalTime = 0;
 		for (int i = 0; i < n; ++i) {
 			long time = testScan(reader, false, info, ParserLanguage.CPP);
@@ -88,7 +87,7 @@ public class Scanner2SpeedTest extends SpeedTest2 {
 	 */
 	protected long testScan(CodeReader reader, boolean quick, IScannerInfo info, ParserLanguage lang) throws Exception {
 		ParserMode mode = quick ? ParserMode.QUICK_PARSE : ParserMode.COMPLETE_PARSE;
-		Scanner2 scanner = createScanner(reader, info, mode, lang, CALLBACK, null, Collections.EMPTY_LIST ); 
+		DOMScanner scanner = createScanner(reader, info, mode, lang, CALLBACK, null, Collections.EMPTY_LIST ); 
 		long startTime = System.currentTimeMillis();
 		int count = 0;
 		try {

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/SpeedTest2.java

@@ -32,7 +32,8 @@ import org.eclipse.cdt.core.parser.ParserFactoryError;
 import org.eclipse.cdt.core.parser.ParserLanguage;
 import org.eclipse.cdt.core.parser.ParserMode;
 import org.eclipse.cdt.core.parser.ScannerInfo;
-import org.eclipse.cdt.internal.core.parser.scanner2.Scanner2;
+import org.eclipse.cdt.internal.core.parser.scanner2.DOMScanner;
+import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory;
 
 // A test that just calculates the speed of the parser
 // Eventually, we'll peg a max time and fail the test if it exceeds it
@@ -87,7 +88,7 @@ public class SpeedTest2 extends TestCase {
 		return totalTime;
 	}
 
-    public static Scanner2 createScanner( CodeReader code, IScannerInfo config, ParserMode mode, ParserLanguage language, ISourceElementRequestor requestor, IParserLogService log, List workingCopies ) throws ParserFactoryError
+    public static DOMScanner createScanner( CodeReader code, IScannerInfo config, ParserMode mode, ParserLanguage language, ISourceElementRequestor requestor, IParserLogService log, List workingCopies ) throws ParserFactoryError
     {
     	if( config == null ) throw new ParserFactoryError( ParserFactoryError.Kind.NULL_CONFIG );
     	if( language == null ) throw new ParserFactoryError( ParserFactoryError.Kind.NULL_LANGUAGE );
@@ -99,7 +100,7 @@ public class SpeedTest2 extends TestCase {
 		    configuration = new GCCScannerExtensionConfiguration();
 		else
 		    configuration = new GPPScannerExtensionConfiguration();
-		return new Scanner2( code, config, ourRequestor, ourMode, language, logService, workingCopies, configuration );
+		return new DOMScanner( code, config, ourMode, language, logService, configuration, FileCodeReaderFactory.getInstance() );
     }
 
 	private static final ISourceElementRequestor CALLBACK = new NullSourceElementRequestor();
@@ -121,15 +122,16 @@ public class SpeedTest2 extends TestCase {
 			return mingwScannerInfo(false);
 	}
 	
-	private IScannerInfo msvcScannerInfo(boolean quick) {
+	protected IScannerInfo msvcScannerInfo(boolean quick) {
 		if( quick )
 			return new ScannerInfo();
 		Map definitions = new Hashtable();
 		//definitions.put( "__GNUC__", "3" );  //$NON-NLS-1$ //$NON-NLS-2$
 
 		String [] includePaths = new String[] {
-			"C:\\Program Files\\Microsoft SDK\\Include",
+			"C:\\Program Files\\Microsoft Visual Studio\\VC98\\Include"
-			"C:\\Program Files\\Microsoft Visual C++ Toolkit 2003\\include"
+//			"C:\\Program Files\\Microsoft Platform SDK\\Include",
+//			"C:\\Program Files\\Microsoft Visual C++ Toolkit 2003\\include"
 		};
 		return new ScannerInfo( definitions, includePaths );
 	}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/CPreprocessor.java

@@ -197,7 +197,7 @@ public class CPreprocessor implements ILexerLog, IScanner {
             IScannerExtensionConfiguration configuration, ICodeReaderFactory readerFactory) {
         fLanguage= language;
         fLog = log;
-        fCheckNumbers= true; // mstodo room for optimization.
+        fCheckNumbers= true; 
         fAdditionalNumericLiteralSuffixes= nonNull(configuration.supportAdditionalNumericLiteralSuffixes());
         fLexOptions.fSupportDollarInitializers= configuration.support$InIdentifiers();
         fLexOptions.fSupportMinAndMax = configuration.supportMinAndMaxOperators();

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Lexer.java

@@ -210,31 +210,67 @@ final public class Lexer {
 	 * @throws OffsetLimitReachedException when completion is requested in a literal or an header-name.
 	 */
 	public Token nextDirective() throws OffsetLimitReachedException {
-		Token t= fToken;
+		final Token t= fToken;
 		boolean haveNL= t==null || t.getType() == tNEWLINE;
-		loop: while(true) {
+		while(true) {
-			t= fetchToken();
+			final boolean hadNL= haveNL;
-			if (haveNL) {
-				switch(t.getType()) {
-				case tEND_OF_INPUT:
-				case IToken.tPOUND:
-					break loop;
-				}
 			haveNL= false;
-			}
+			final int start= fOffset;
-			else {
+			final int c= fCharPhase3;
-				switch(t.getType()) {
+			final int d= nextCharPhase3();
-				case tEND_OF_INPUT:
+			switch(c) {
-					break loop;
+			case END_OF_INPUT:
-				case tNEWLINE:
+				fToken= newToken(Lexer.tEND_OF_INPUT, start);
+				return fToken;
+			case '\n':
 				haveNL= true;
-					break;
+				fInsideIncludeDirective= false;
+				continue;
+			case ' ':
+			case '\t':
+			case 0xb:  // vertical tab
+			case '\f': 
+			case '\r':
+				haveNL= hadNL;
+				continue;
+				
+			case '"':
+				stringLiteral(start, false);
+				continue;
+
+			case '\'':
+				charLiteral(start, false);
+				continue;
+
+			case '/':
+				switch (d) {
+				case '/':
+					nextCharPhase3();
+					lineComment(start);
+					continue; 
+				case '*':
+					nextCharPhase3();
+					blockComment(start);
+					continue;
+				}
+				continue;
+
+			case '#':
+				if (d == '#') {
+					nextCharPhase3();
+					continue;
+				}
+				if (hadNL) {
+					fToken= newToken(IToken.tPOUND, start);
+					return fToken;
+				}
+				continue;
+
+			default:
+				continue;
 			}
 		}
 	}
-		fToken= t;
-		return t;
-	}
 	
 	/**
 	 * Computes the next token.