From e0ba2b4f8441f7de45d592d4bea390af9302a824 Mon Sep 17 00:00:00 2001
From: Markus Schorn <markus.schorn@windriver.com>
Date: Tue, 6 Nov 2007 16:41:44 +0000
Subject: [PATCH] Parser tests using CPreprocessor

---
 .../core/parser/tests/ast2/AST2BaseTest.java  | 18 +++-
 .../parser/tests/ast2/AST2CPPSpecTest.java    | 12 +--
 .../ast2/AST2SelectionParseBaseTest.java      | 12 +--
 .../parser/tests/ast2/AST2SpecBaseTest.java   | 16 +---
 .../cdt/core/parser/tests/ast2/AST2Tests.java |  3 +-
 .../tests/ast2/CompleteParser2Tests.java      | 12 +--
 .../tests/ast2/DOMLocationMacroTests.java     |  9 +-
 .../parser/tests/ast2/DOMLocationTests.java   | 14 ++-
 .../ast2/DOMPreprocessorInformationTest.java  | 12 +--
 .../parser/tests/ast2/DOMScannerTests.java    | 27 ++++--
 .../parser/tests/ast2/QuickParser2Tests.java  | 12 +--
 .../tests/prefix/CompletionTestBase.java      | 13 +--
 .../tests/scanner/PreprocessorTests.java      | 86 +++++++++++++++++++
 13 files changed, 156 insertions(+), 90 deletions(-)

diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2BaseTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2BaseTest.java
index 12bc13dd865..53d2920cd27 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2BaseTest.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2BaseTest.java
@@ -56,6 +56,7 @@ import org.eclipse.cdt.core.dom.parser.cpp.ICPPParserExtensionConfiguration;
 import org.eclipse.cdt.core.parser.CodeReader;
 import org.eclipse.cdt.core.parser.IParserLogService;
 import org.eclipse.cdt.core.parser.IScanner;
+import org.eclipse.cdt.core.parser.IScannerInfo;
 import org.eclipse.cdt.core.parser.NullLogService;
 import org.eclipse.cdt.core.parser.ParserLanguage;
 import org.eclipse.cdt.core.parser.ParserMode;
@@ -68,6 +69,7 @@ import org.eclipse.cdt.internal.core.dom.parser.c.GNUCSourceParser;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPVisitor;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.GNUCPPSourceParser;
 import org.eclipse.cdt.internal.core.parser.ParserException;
+import org.eclipse.cdt.internal.core.parser.scanner.CPreprocessor;
 import org.eclipse.cdt.internal.core.parser.scanner2.DOMScanner;
 import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory;
 
@@ -77,6 +79,7 @@ import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory;
 public class AST2BaseTest extends BaseTestCase {
 	
     private static final IParserLogService NULL_LOG = new NullLogService();
+    protected boolean fUsesCPreprocessor= false;
 
     public AST2BaseTest() {
     	super();
@@ -106,6 +109,7 @@ public class AST2BaseTest extends BaseTestCase {
     protected IASTTranslationUnit parse( String code, ParserLanguage lang, boolean useGNUExtensions, boolean expectNoProblems , boolean parseComments) throws ParserException {
         IScanner scanner = createScanner(new CodeReader(code.toCharArray()), lang, ParserMode.COMPLETE_PARSE, 
         		new ScannerInfo(), parseComments);
+        fUsesCPreprocessor= scanner instanceof CPreprocessor;
         ISourceCodeParser parser2 = null;
         if( lang == ParserLanguage.CPP )
         {
@@ -151,14 +155,22 @@ public class AST2BaseTest extends BaseTestCase {
     }
 
 	public static IScanner createScanner(CodeReader codeReader, ParserLanguage lang, ParserMode mode,
-			ScannerInfo scannerInfo, boolean parseComments) {
+			IScannerInfo scannerInfo, boolean parseComments) {
 		IScannerExtensionConfiguration configuration = null;
         if( lang == ParserLanguage.C )
             configuration = new GCCScannerExtensionConfiguration();
         else
             configuration = new GPPScannerExtensionConfiguration();
-        IScanner scanner = new DOMScanner( codeReader, scannerInfo, mode, lang, NULL_LOG, configuration, FileCodeReaderFactory.getInstance() );
-        scanner.setScanComments(parseComments);
+        IScanner scanner;
+        if (CPreprocessor.PROP_VALUE.equals(System.getProperty("scanner"))) {
+        	scanner= new CPreprocessor(codeReader, scannerInfo, lang, NULL_LOG, configuration, 
+        			FileCodeReaderFactory.getInstance());
+        }
+        else {
+        	scanner = new DOMScanner( codeReader, scannerInfo, mode, lang, NULL_LOG, configuration, 
+        			FileCodeReaderFactory.getInstance() );
+        	scanner.setScanComments(parseComments);
+        }
 		return scanner;
 	}
 
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPSpecTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPSpecTest.java
index 0c7ed94ad6e..a23d8688bc6 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPSpecTest.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CPPSpecTest.java
@@ -6572,7 +6572,7 @@ public class AST2CPPSpecTest extends AST2SpecBaseTest {
 		buffer.append("int i = cobj.c * 100; // value of cobj.c is unspecified\n"); //$NON-NLS-1$
 		buffer.append("cptr->c = 1;\n"); //$NON-NLS-1$
 		buffer.append("cout << cobj.c * 100 // value of cobj.c is unspecified\n"); //$NON-NLS-1$
-		buffer.append("<< '\n';\n"); //$NON-NLS-1$
+		buffer.append("<< '\\n';\n"); //$NON-NLS-1$
 		buffer.append("}\n"); //$NON-NLS-1$
 		parse(buffer.toString(), ParserLanguage.CPP, false, 0);
 	}
@@ -9734,7 +9734,7 @@ public class AST2CPPSpecTest extends AST2SpecBaseTest {
 		buffer.append("void printall()\n"); //$NON-NLS-1$
 		buffer.append("{\n"); //$NON-NLS-1$
 		buffer.append("for (int i = 0; i<cnt; i++)\n"); //$NON-NLS-1$
-		buffer.append("cout << p[i] << '\n';\n"); //$NON-NLS-1$
+		buffer.append("cout << p[i] << '\\n';\n"); //$NON-NLS-1$
 		buffer.append("}\n"); //$NON-NLS-1$
 		buffer.append("// ...\n"); //$NON-NLS-1$
 		buffer.append("};\n"); //$NON-NLS-1$
@@ -11645,9 +11645,9 @@ public class AST2CPPSpecTest extends AST2SpecBaseTest {
 	
 	/**
 	 [--Start Example(CPP 16.2-8):
-	#if VERSION = = 1
+	#if VERSION == 1
 	#define INCFILE "vers1.h"
-	#elif VERSION = = 2
+	#elif VERSION == 2
 	#define INCFILE "vers2.h" // and so on
 	#else
 	#define INCFILE "versN.h"
@@ -11656,9 +11656,9 @@ public class AST2CPPSpecTest extends AST2SpecBaseTest {
 	 */
 	public void test16_2s8() throws Exception {
 		StringBuffer buffer = new StringBuffer();
-		buffer.append("#if VERSION = = 1\n"); //$NON-NLS-1$
+		buffer.append("#if VERSION == 1\n"); //$NON-NLS-1$
 		buffer.append("#define INCFILE \"vers1.h\"\n"); //$NON-NLS-1$
-		buffer.append("#elif VERSION = = 2\n"); //$NON-NLS-1$
+		buffer.append("#elif VERSION == 2\n"); //$NON-NLS-1$
 		buffer.append("#define INCFILE \"vers2.h\" // and so on\n"); //$NON-NLS-1$
 		buffer.append("#else\n"); //$NON-NLS-1$
 		buffer.append("#define INCFILE \"versN.h\"\n"); //$NON-NLS-1$
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2SelectionParseBaseTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2SelectionParseBaseTest.java
index 1dab1ee80a7..d383f308005 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2SelectionParseBaseTest.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2SelectionParseBaseTest.java
@@ -14,15 +14,12 @@ import org.eclipse.cdt.core.dom.CDOM;
 import org.eclipse.cdt.core.dom.IASTServiceProvider.UnsupportedDialectException;
 import org.eclipse.cdt.core.dom.ast.IASTNode;
 import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit;
-import org.eclipse.cdt.core.dom.parser.IScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.ISourceCodeParser;
 import org.eclipse.cdt.core.dom.parser.c.ANSICParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.c.GCCParserExtensionConfiguration;
-import org.eclipse.cdt.core.dom.parser.c.GCCScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.c.ICParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.ANSICPPParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.GPPParserExtensionConfiguration;
-import org.eclipse.cdt.core.dom.parser.cpp.GPPScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.ICPPParserExtensionConfiguration;
 import org.eclipse.cdt.core.parser.CodeReader;
 import org.eclipse.cdt.core.parser.IParserLogService;
@@ -37,8 +34,6 @@ import org.eclipse.cdt.internal.core.dom.parser.c.GNUCSourceParser;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPVisitor;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.GNUCPPSourceParser;
 import org.eclipse.cdt.internal.core.parser.ParserException;
-import org.eclipse.cdt.internal.core.parser.scanner2.DOMScanner;
-import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory;
 import org.eclipse.core.resources.IFile;
 
 /**
@@ -87,12 +82,7 @@ public class AST2SelectionParseBaseTest extends FileBasePluginTest {
         CodeReader codeReader = new CodeReader(code
                 .toCharArray());
         ScannerInfo scannerInfo = new ScannerInfo();
-        IScannerExtensionConfiguration configuration = null;
-        if( lang == ParserLanguage.C )
-            configuration = new GCCScannerExtensionConfiguration();
-        else
-            configuration = new GPPScannerExtensionConfiguration();
-        IScanner scanner = new DOMScanner( codeReader, scannerInfo, ParserMode.COMPLETE_PARSE, lang, NULL_LOG, configuration, FileCodeReaderFactory.getInstance() );
+        IScanner scanner= AST2BaseTest.createScanner(codeReader, lang, ParserMode.COMPLETE_PARSE, scannerInfo, false);
         
         ISourceCodeParser parser2 = null;
         if( lang == ParserLanguage.CPP )
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2SpecBaseTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2SpecBaseTest.java
index c0de688eae0..b04d449de58 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2SpecBaseTest.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2SpecBaseTest.java
@@ -13,21 +13,20 @@ package org.eclipse.cdt.core.parser.tests.ast2;
 import java.util.ArrayList;
 import java.util.List;
 
+import junit.framework.TestCase;
+
 import org.eclipse.cdt.core.dom.ast.IASTName;
 import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit;
 import org.eclipse.cdt.core.dom.ast.IBinding;
 import org.eclipse.cdt.core.dom.ast.IProblemBinding;
 import org.eclipse.cdt.core.dom.ast.c.CASTVisitor;
 import org.eclipse.cdt.core.dom.ast.cpp.CPPASTVisitor;
-import org.eclipse.cdt.core.dom.parser.IScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.ISourceCodeParser;
 import org.eclipse.cdt.core.dom.parser.c.ANSICParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.c.GCCParserExtensionConfiguration;
-import org.eclipse.cdt.core.dom.parser.c.GCCScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.c.ICParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.ANSICPPParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.GPPParserExtensionConfiguration;
-import org.eclipse.cdt.core.dom.parser.cpp.GPPScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.ICPPParserExtensionConfiguration;
 import org.eclipse.cdt.core.parser.CodeReader;
 import org.eclipse.cdt.core.parser.IParserLogService;
@@ -41,10 +40,6 @@ import org.eclipse.cdt.internal.core.dom.parser.c.GNUCSourceParser;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPVisitor;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.GNUCPPSourceParser;
 import org.eclipse.cdt.internal.core.parser.ParserException;
-import org.eclipse.cdt.internal.core.parser.scanner2.DOMScanner;
-import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory;
-
-import junit.framework.TestCase;
 
 /**
  * @author dsteffle
@@ -100,12 +95,7 @@ public class AST2SpecBaseTest extends TestCase {
 	
 	private IASTTranslationUnit parse(CodeReader codeReader, ParserLanguage lang, boolean useGNUExtensions, boolean expectNoProblems, boolean checkBindings, int expectedProblemBindings) throws ParserException {
         ScannerInfo scannerInfo = new ScannerInfo();
-        IScannerExtensionConfiguration configuration = null;
-        if( lang == ParserLanguage.C )
-            configuration = new GCCScannerExtensionConfiguration();
-        else
-            configuration = new GPPScannerExtensionConfiguration();
-        IScanner scanner = new DOMScanner( codeReader, scannerInfo, ParserMode.COMPLETE_PARSE, lang, NULL_LOG, configuration, FileCodeReaderFactory.getInstance() );
+        IScanner scanner= AST2BaseTest.createScanner(codeReader, lang, ParserMode.COMPLETE_PARSE, scannerInfo, false);
         
         ISourceCodeParser parser2 = null;
         if( lang == ParserLanguage.CPP )
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2Tests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2Tests.java
index 9e8f9cb0231..8daae208998 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2Tests.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2Tests.java
@@ -3585,7 +3585,8 @@ public class AST2Tests extends AST2BaseTest {
     	assertEquals("1", macroDefinitions[0].getExpansion());
     	assertEquals("1", macroDefinitions[1].getExpansion());
     	// regression test for #64268 and #71733 which also handle comments
-    	assertEquals("1   + 2", macroDefinitions[2].getExpansion());
+    	String expectExpansion= fUsesCPreprocessor ? "1 + 2" : "1   + 2";
+    	assertEquals(expectExpansion, macroDefinitions[2].getExpansion());
     	assertEquals("(KDebugNum(x))", macroDefinitions[3].getExpansion());
     	assertEquals("{if((DEBUGNUM(a)))p;}", macroDefinitions[4].getExpansion());
     	
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/CompleteParser2Tests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/CompleteParser2Tests.java
index bb98b30c266..e3d99fab181 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/CompleteParser2Tests.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/CompleteParser2Tests.java
@@ -63,15 +63,12 @@ import org.eclipse.cdt.core.dom.ast.cpp.ICPPNamespace;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPReferenceType;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPUsingDeclaration;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPVariable;
-import org.eclipse.cdt.core.dom.parser.IScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.ISourceCodeParser;
 import org.eclipse.cdt.core.dom.parser.c.ANSICParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.c.GCCParserExtensionConfiguration;
-import org.eclipse.cdt.core.dom.parser.c.GCCScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.c.ICParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.ANSICPPParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.GPPParserExtensionConfiguration;
-import org.eclipse.cdt.core.dom.parser.cpp.GPPScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.ICPPParserExtensionConfiguration;
 import org.eclipse.cdt.core.parser.CodeReader;
 import org.eclipse.cdt.core.parser.IScanner;
@@ -85,8 +82,6 @@ import org.eclipse.cdt.internal.core.dom.parser.c.GNUCSourceParser;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPVisitor;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.GNUCPPSourceParser;
 import org.eclipse.cdt.internal.core.parser.ParserException;
-import org.eclipse.cdt.internal.core.parser.scanner2.DOMScanner;
-import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory;
 import org.eclipse.core.runtime.Platform;
 
 /**
@@ -183,13 +178,8 @@ public class CompleteParser2Tests extends BaseTestCase {
         CodeReader codeReader = new CodeReader(code
                 .toCharArray());
         ScannerInfo scannerInfo = new ScannerInfo();
-        IScannerExtensionConfiguration configuration = null;
-        if( lang == ParserLanguage.C )
-            configuration = new GCCScannerExtensionConfiguration();
-        else
-            configuration = new GPPScannerExtensionConfiguration();
         ISourceCodeParser parser2 = null;
-        IScanner scanner = new DOMScanner( codeReader, scannerInfo, ParserMode.COMPLETE_PARSE, lang, NULL_LOG, configuration, FileCodeReaderFactory.getInstance() );
+        IScanner scanner= AST2BaseTest.createScanner(codeReader, lang, ParserMode.COMPLETE_PARSE, scannerInfo, false);
         if (lang == ParserLanguage.CPP) {
             ICPPParserExtensionConfiguration config = null;
             if (gcc)
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMLocationMacroTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMLocationMacroTests.java
index b879a6a5936..9c9f053ea08 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMLocationMacroTests.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMLocationMacroTests.java
@@ -87,7 +87,7 @@ public class DOMLocationMacroTests extends AST2BaseTest {
             IASTPreprocessorObjectStyleMacroDefinition fromExpansion = (IASTPreprocessorObjectStyleMacroDefinition) expansion.getMacroDefinition();
             assertEqualsMacros( fromExpansion, ABC );
             assertEquals( expansion.getNodeOffset(), 0 );
-            assertEquals( expansion.getNodeLength(), 3 );
+            assertEquals( fUsesCPreprocessor ? 2 : 3, expansion.getNodeLength() );
             IASTNodeLocation [] macroLocation = expansion.getExpansionLocations();
             assertEquals( macroLocation.length, 1 );
             assertTrue( macroLocation[0] instanceof IASTFileLocation );
@@ -98,7 +98,7 @@ public class DOMLocationMacroTests extends AST2BaseTest {
             IASTNodeLocation [] nameLocations = n.getNodeLocations();
             assertEquals( nameLocations.length, 1 );
             final IASTMacroExpansion nodeLocation = (IASTMacroExpansion) nameLocations[0];
-            assertEquals( nodeLocation.getNodeOffset(), 2 );
+            assertEquals( nodeLocation.getNodeOffset(), fUsesCPreprocessor ? 1 : 2 );
             assertEquals( nodeLocation.getNodeLength(), 1 );
             
             assertEquals( nodeLocation.getExpansionLocations()[0].getNodeOffset(), macroLocation[0].getNodeOffset() );
@@ -132,7 +132,7 @@ public class DOMLocationMacroTests extends AST2BaseTest {
             IASTMacroExpansion expansion = (IASTMacroExpansion) declSpecLocations[0];
             assertEqualsMacros( defXYZ, expansion.getMacroDefinition() );
             assertEquals( expansion.getNodeOffset(), 0 );
-            assertEquals( expansion.getNodeLength(), "const".length() );
+            assertEquals( expansion.getNodeLength(), fUsesCPreprocessor ? 1 : "const".length() );
             IASTNodeLocation [] expansionLocations = expansion.getExpansionLocations();
             assertEquals( expansionLocations.length, 1 );
             assertTrue( expansionLocations[0] instanceof IASTFileLocation );
@@ -170,7 +170,8 @@ public class DOMLocationMacroTests extends AST2BaseTest {
             final IASTPreprocessorMacroDefinition C_PO2 = mac_loc.getMacroDefinition();
             assertEqualsMacros( C_PO, C_PO2 );
             assertEquals( 0, mac_loc.getNodeOffset());
-            assertEquals( 4+ C_PO.getExpansion().length() + XYZ.getExpansion().length() + PO.getExpansion().length(), mac_loc.getNodeLength() );
+            assertEquals( fUsesCPreprocessor ? 2 :
+            		4+ C_PO.getExpansion().length() + XYZ.getExpansion().length() + PO.getExpansion().length(), mac_loc.getNodeLength() );
             IASTFileLocation end_loc = (IASTFileLocation) locations[2];
             assertEquals( code.indexOf( " var"), end_loc.getNodeOffset() );  //$NON-NLS-1$
             assertEquals( " var;".length(), end_loc.getNodeLength() ); //$NON-NLS-1$
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMLocationTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMLocationTests.java
index 761f654a9d0..574d0473e7e 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMLocationTests.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMLocationTests.java
@@ -92,13 +92,13 @@ public class DOMLocationTests extends AST2BaseTest {
             assertEquals(nodeLocations.length, 1);
             assertTrue(nodeLocations[0] instanceof IASTFileLocation);
             IASTFileLocation fileLocation = ((IASTFileLocation) nodeLocations[0]);
-            assertEquals(fileLocation.getFileName(), _TEXT_); //$NON-NLS-1$
+            assertEquals(fileLocation.getFileName(), _TEXT_); 
             assertEquals(fileLocation.getNodeOffset(), 0);
             assertEquals(fileLocation.getNodeLength(), 6);
             IASTNodeLocation[] tuLocations = tu.getNodeLocations();
             assertEquals(tuLocations.length, nodeLocations.length);
             assertEquals(fileLocation.getFileName(),
-                    ((IASTFileLocation) tuLocations[0]).getFileName()); //$NON-NLS-1$
+                    ((IASTFileLocation) tuLocations[0]).getFileName()); 
             assertEquals(fileLocation.getNodeOffset(), tuLocations[0]
                     .getNodeOffset());
             assertEquals(fileLocation.getNodeLength(), tuLocations[0]
@@ -119,7 +119,7 @@ public class DOMLocationTests extends AST2BaseTest {
             assertEquals(nodeLocations.length, 1);
             assertTrue(nodeLocations[0] instanceof IASTFileLocation);
             IASTFileLocation fileLocation = ((IASTFileLocation) nodeLocations[0]);
-            assertEquals(fileLocation.getFileName(), _TEXT_); //$NON-NLS-1$
+            assertEquals(fileLocation.getFileName(), _TEXT_); 
             assertEquals(fileLocation.getNodeOffset(), 0);
             assertEquals(fileLocation.getNodeLength(), code.indexOf(";") + 1); //$NON-NLS-1$
             IASTDeclarator[] declarators = declaration.getDeclarators();
@@ -578,7 +578,7 @@ public class DOMLocationTests extends AST2BaseTest {
             assertEquals( 2, decls.length);
             assertEquals( 1, statements.length);
             assertEquals( 1, problems.length);
-            assertSoleLocation(problems[0], code, "z");
+            assertSoleLocation(problems[0], code, fUsesCPreprocessor ? "nix(y," : "z");
             assertSoleLocation( decls[1], code, "int x;"); 
         }
     }
@@ -593,13 +593,11 @@ public class DOMLocationTests extends AST2BaseTest {
                 : null) {
             IASTTranslationUnit tu = parse(code, p, false, false);
             IASTDeclaration[] decls= tu.getDeclarations();
-            IASTPreprocessorStatement [] statements = tu.getAllPreprocessorStatements();
             IASTProblem[] problems = tu.getPreprocessorProblems();
             assertEquals( 1, decls.length);
-            assertEquals( 1, statements.length);
             assertEquals( 2, problems.length);
             assertSoleLocation(problems[0], code, "#include \"\"");
-            assertSoleLocation(problems[1], code, "else");
+            assertSoleLocation(problems[1], code, fUsesCPreprocessor ? "#else" : "else");
             assertSoleLocation( decls[0], code, "int x;"); 
         }
     }
@@ -720,6 +718,6 @@ public class DOMLocationTests extends AST2BaseTest {
         assertEquals(IASTProblem.PREPROCESSOR_INVALID_DIRECTIVE, problems[2].getID());
         assertSoleLocation(problems[0], code, "#import \"include_once.h\"");
         assertSoleLocation(problems[1], code, "\"deprecated include\"");
-        assertSoleLocation(problems[2], code, "invalid");
+        assertSoleLocation(problems[2], code, fUsesCPreprocessor ? "#invalid" : "invalid");        	
     }
 }
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMPreprocessorInformationTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMPreprocessorInformationTest.java
index aef20526926..265a1b235fb 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMPreprocessorInformationTest.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMPreprocessorInformationTest.java
@@ -183,11 +183,13 @@ public class DOMPreprocessorInformationTest extends AST2BaseTest {
 		IASTNodeLocation[] nodeLocations = init.getNodeLocations();
 		assertEquals(1, nodeLocations.length);
 		
-		FunctionMacroExpansionLocation location = (FunctionMacroExpansionLocation) nodeLocations[0];
-		char[][] actualParameters = location.getActualParameters();
-		
-		assertEquals("foo", new String(actualParameters[0]));
-		assertEquals("bar", new String(actualParameters[1]));
+		if (!fUsesCPreprocessor) {
+			FunctionMacroExpansionLocation location = (FunctionMacroExpansionLocation) nodeLocations[0];
+			char[][] actualParameters = location.getActualParameters();
+
+			assertEquals("foo", new String(actualParameters[0]));
+			assertEquals("bar", new String(actualParameters[1]));
+		}
 	}
 	
 	// #ifdef xxx
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMScannerTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMScannerTests.java
index d8f4623c997..30faefd3b74 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMScannerTests.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/DOMScannerTests.java
@@ -23,11 +23,15 @@ import junit.framework.TestSuite;
 
 import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIncludeStatement;
 import org.eclipse.cdt.core.dom.ast.IASTProblem;
+import org.eclipse.cdt.core.dom.parser.IScannerExtensionConfiguration;
+import org.eclipse.cdt.core.dom.parser.c.GCCScannerExtensionConfiguration;
+import org.eclipse.cdt.core.dom.parser.cpp.GPPScannerExtensionConfiguration;
 import org.eclipse.cdt.core.parser.CodeReader;
 import org.eclipse.cdt.core.parser.EndOfFileException;
 import org.eclipse.cdt.core.parser.IProblem;
 import org.eclipse.cdt.core.parser.ISourceElementRequestor;
 import org.eclipse.cdt.core.parser.IToken;
+import org.eclipse.cdt.core.parser.NullLogService;
 import org.eclipse.cdt.core.parser.NullSourceElementRequestor;
 import org.eclipse.cdt.core.parser.ParserLanguage;
 import org.eclipse.cdt.core.parser.ParserMode;
@@ -35,6 +39,7 @@ import org.eclipse.cdt.core.parser.ScannerInfo;
 import org.eclipse.cdt.core.parser.util.CharArrayUtils;
 import org.eclipse.cdt.core.testplugin.util.BaseTestCase;
 import org.eclipse.cdt.internal.core.parser.scanner2.DOMScanner;
+import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory;
 import org.eclipse.cdt.internal.core.parser.scanner2.FunctionStyleMacro;
 import org.eclipse.cdt.internal.core.parser.scanner2.ILocationResolver;
 import org.eclipse.cdt.internal.core.parser.scanner2.ObjectStyleMacro;
@@ -62,14 +67,24 @@ public class DOMScannerTests extends BaseTestCase {
 		initializeScanner(input, ParserMode.COMPLETE_PARSE);
 	}
 
-	protected void initializeScanner(String input, ParserLanguage lang) throws IOException {
-		fScanner= (DOMScanner) AST2BaseTest.createScanner(new CodeReader(input.toCharArray()), lang, 
-				ParserMode.COMPLETE_PARSE, new ScannerInfo(), false);
+	protected void initializeScanner(String input, ParserLanguage lang) {
+		initializeScanner(input, ParserMode.COMPLETE_PARSE, lang);
 	}
 	
-	protected void initializeScanner(String input, ParserMode mode) throws IOException {
-		fScanner= (DOMScanner) AST2BaseTest.createScanner(new CodeReader(input.toCharArray()), ParserLanguage.CPP, 
-				mode, new ScannerInfo(), false);
+	protected void initializeScanner(String input, ParserMode mode) {
+		initializeScanner(input, mode, ParserLanguage.CPP); 
+	}
+	
+	protected void initializeScanner(String input, ParserMode mode, ParserLanguage lang) {
+		CodeReader codeReader=new CodeReader(input.toCharArray());
+		IScannerExtensionConfiguration configuration = null;
+		if( lang == ParserLanguage.C )
+			configuration = new GCCScannerExtensionConfiguration();
+		else
+			configuration = new GPPScannerExtensionConfiguration();
+		fScanner= new DOMScanner( codeReader, new ScannerInfo(), mode, lang, 
+				new NullLogService(), configuration, FileCodeReaderFactory.getInstance() );
+		fScanner.setScanComments(false);
 	}
 	
 	protected int fullyTokenize() throws Exception {
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/QuickParser2Tests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/QuickParser2Tests.java
index 5505c1173f0..d4cc3a774e1 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/QuickParser2Tests.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/QuickParser2Tests.java
@@ -18,15 +18,12 @@ import junit.framework.TestCase;
 
 import org.eclipse.cdt.core.dom.ast.IASTProblem;
 import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit;
-import org.eclipse.cdt.core.dom.parser.IScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.ISourceCodeParser;
 import org.eclipse.cdt.core.dom.parser.c.ANSICParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.c.GCCParserExtensionConfiguration;
-import org.eclipse.cdt.core.dom.parser.c.GCCScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.c.ICParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.ANSICPPParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.GPPParserExtensionConfiguration;
-import org.eclipse.cdt.core.dom.parser.cpp.GPPScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.ICPPParserExtensionConfiguration;
 import org.eclipse.cdt.core.parser.CodeReader;
 import org.eclipse.cdt.core.parser.IScanner;
@@ -40,8 +37,6 @@ import org.eclipse.cdt.internal.core.dom.parser.c.GNUCSourceParser;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPVisitor;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.GNUCPPSourceParser;
 import org.eclipse.cdt.internal.core.parser.ParserException;
-import org.eclipse.cdt.internal.core.parser.scanner2.DOMScanner;
-import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory;
 
 /**
  * @author jcamelon
@@ -1363,12 +1358,7 @@ public class QuickParser2Tests extends TestCase {
 
         CodeReader codeReader = new CodeReader( code.toCharArray() );
         IScannerInfo scannerInfo = new ScannerInfo();
-        IScannerExtensionConfiguration configuration = null;
-        if( lang == ParserLanguage.C )
-            configuration = new GCCScannerExtensionConfiguration();
-        else
-            configuration = new GPPScannerExtensionConfiguration();
-        IScanner scanner = new DOMScanner( codeReader, scannerInfo, ParserMode.COMPLETE_PARSE, lang, NULL_LOG, configuration, FileCodeReaderFactory.getInstance() );
+        IScanner scanner= AST2BaseTest.createScanner(codeReader, lang, ParserMode.COMPLETE_PARSE, scannerInfo, false);
         ISourceCodeParser parser2 = null;
         if (lang == ParserLanguage.CPP) {
             ICPPParserExtensionConfiguration config = null;
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/prefix/CompletionTestBase.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/prefix/CompletionTestBase.java
index ff42f62472c..3eee4713942 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/prefix/CompletionTestBase.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/prefix/CompletionTestBase.java
@@ -17,15 +17,12 @@ import junit.framework.TestCase;
 
 import org.eclipse.cdt.core.dom.ast.IASTCompletionNode;
 import org.eclipse.cdt.core.dom.ast.IBinding;
-import org.eclipse.cdt.core.dom.parser.IScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.ISourceCodeParser;
 import org.eclipse.cdt.core.dom.parser.c.ANSICParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.c.GCCParserExtensionConfiguration;
-import org.eclipse.cdt.core.dom.parser.c.GCCScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.c.ICParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.ANSICPPParserExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.GPPParserExtensionConfiguration;
-import org.eclipse.cdt.core.dom.parser.cpp.GPPScannerExtensionConfiguration;
 import org.eclipse.cdt.core.dom.parser.cpp.ICPPParserExtensionConfiguration;
 import org.eclipse.cdt.core.parser.CodeReader;
 import org.eclipse.cdt.core.parser.IParserLogService;
@@ -34,11 +31,10 @@ import org.eclipse.cdt.core.parser.NullLogService;
 import org.eclipse.cdt.core.parser.ParserLanguage;
 import org.eclipse.cdt.core.parser.ParserMode;
 import org.eclipse.cdt.core.parser.ScannerInfo;
+import org.eclipse.cdt.core.parser.tests.ast2.AST2BaseTest;
 import org.eclipse.cdt.internal.core.dom.parser.c.GNUCSourceParser;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.GNUCPPSourceParser;
 import org.eclipse.cdt.internal.core.parser.ParserException;
-import org.eclipse.cdt.internal.core.parser.scanner2.DOMScanner;
-import org.eclipse.cdt.internal.core.parser.scanner2.FileCodeReaderFactory;
 
 public class CompletionTestBase extends TestCase {
 
@@ -47,12 +43,7 @@ public class CompletionTestBase extends TestCase {
 	protected IASTCompletionNode getCompletionNode(String code, ParserLanguage lang, boolean useGNUExtensions) throws ParserException {
         CodeReader codeReader = new CodeReader(code.toCharArray());
         ScannerInfo scannerInfo = new ScannerInfo();
-        IScannerExtensionConfiguration configuration = null;
-        if( lang == ParserLanguage.C )
-            configuration = new GCCScannerExtensionConfiguration();
-        else
-            configuration = new GPPScannerExtensionConfiguration();
-        IScanner scanner = new DOMScanner( codeReader, scannerInfo, ParserMode.COMPLETE_PARSE, lang, NULL_LOG, configuration, FileCodeReaderFactory.getInstance() );
+        IScanner scanner= AST2BaseTest.createScanner(codeReader, lang, ParserMode.COMPLETE_PARSE, scannerInfo, false);
         
         ISourceCodeParser parser = null;
         if( lang == ParserLanguage.CPP )
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTests.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTests.java
index 685a5f81146..61fdb2e0ea1 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTests.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner/PreprocessorTests.java
@@ -452,6 +452,7 @@ public class PreprocessorTests extends PreprocessorTestsBase {
     // #define tp2(x,y,z) [ x ## y ## z ]
     // #define tstr1(x,y) [#x#y]
     // #define tstr2(x,y) [ #x #y ]
+    // #define spaceBeforeStr(x) a #x b
     // xstr(nospace);
     // xstr(space);
     // xstr(tp1(a b, c d , e f));
@@ -460,6 +461,7 @@ public class PreprocessorTests extends PreprocessorTestsBase {
     // xstr(tp2(a-b, c-d , e-f));
     // xstr(tstr1(a b, c d));
     // xstr(tstr2(a b, c d));
+    // xstr(spaceBeforeStr(c));
     public void testSpaceInStringify() throws Exception {
 		initializeScanner();
 		validateString("fvalfval");
@@ -486,7 +488,91 @@ public class PreprocessorTests extends PreprocessorTestsBase {
 		validateString("[ \\\"a b\\\" \\\"c d\\\" ]");
 		validateToken(IToken.tSEMI);
 
+		validateString("a \\\"c\\\" b");
+		validateToken(IToken.tSEMI);
+
 		validateEOF();
 		validateProblemCount(0);
     }
+
+    // #define empty
+    // #define paste1(y) x##y z
+    // #define paste2(x) x##empty z
+    // paste1();
+    // paste1(empty);
+    // paste2();
+    // paste2(empty);
+    // paste2(a);
+    public void testTokenPasteWithEmptyParam() throws Exception {
+		initializeScanner();
+		validateIdentifier("x");
+		validateIdentifier("z");
+		validateToken(IToken.tSEMI);
+		
+		validateIdentifier("xempty");
+		validateIdentifier("z");
+		validateToken(IToken.tSEMI);
+		
+		validateIdentifier("z");
+		validateToken(IToken.tSEMI);
+		
+		validateIdentifier("emptyempty");
+		validateIdentifier("z");
+		validateToken(IToken.tSEMI);
+
+		validateIdentifier("aempty");
+		validateIdentifier("z");
+		validateToken(IToken.tSEMI);
+
+		validateEOF();
+		validateProblemCount(0);    	
+    }
+    
+    // #define empty
+    // #define paste1(y) x##y z
+    // #define paste2(x) x##empty z
+    // paste1();
+    // paste1(empty);
+    // paste2();
+    // paste2(empty);
+    // paste2(a);
+    public void testSpacesBeforeStringify() throws Exception {
+		initializeScanner();
+		validateIdentifier("x");
+		validateIdentifier("z");
+		validateToken(IToken.tSEMI);
+		
+		validateIdentifier("xempty");
+		validateIdentifier("z");
+		validateToken(IToken.tSEMI);
+		
+		validateIdentifier("z");
+		validateToken(IToken.tSEMI);
+		
+		validateIdentifier("emptyempty");
+		validateIdentifier("z");
+		validateToken(IToken.tSEMI);
+
+		validateIdentifier("aempty");
+		validateIdentifier("z");
+		validateToken(IToken.tSEMI);
+
+		validateEOF();
+		validateProblemCount(0);    	
+    }
+
+    // #define paste(x,y,z) x##y##z
+    // paste(a,b,c);
+    // paste(1,2,3);
+    public void testTokenPasteChain() throws Exception {
+    	initializeScanner();
+		validateIdentifier("abc");
+		validateToken(IToken.tSEMI);
+
+		validateInteger("123");
+		validateToken(IToken.tSEMI);
+
+		validateEOF();
+		validateProblemCount(0);    	    	
+    }
 }