From ba9fc2e4628e28f076286adfc35feebd6ce77a24 Mon Sep 17 00:00:00 2001
From: John Camelon <jcamelon@ca.ibm.com>
Date: Wed, 25 Feb 2004 15:30:21 +0000
Subject: [PATCH] org.eclipse.cdt.core 	Updated Scanner to allow for invalid
 identifier names despite C++'s best efforts at maintaining its honour.

org.eclipse.cdt.core.tests
	Added ScannerTestCase::testGerman().
---
 core/org.eclipse.cdt.core.tests/ChangeLog     |  3 +++
 .../core/parser/tests/BaseScannerTest.java    |  4 ++--
 .../core/parser/tests/ScannerTestCase.java    |  9 +++++++++
 .../parser/ChangeLog-parser                   |  3 +++
 .../internal/core/parser/scanner/Scanner.java | 20 ++++++++++++-------
 5 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/core/org.eclipse.cdt.core.tests/ChangeLog b/core/org.eclipse.cdt.core.tests/ChangeLog
index a518936e0b7..1dce28d7eb8 100644
--- a/core/org.eclipse.cdt.core.tests/ChangeLog
+++ b/core/org.eclipse.cdt.core.tests/ChangeLog
@@ -1,3 +1,6 @@
+2004-02-25 John Camelon
+	Added ScannerTestCase::testGerman().  
+
 2004-02-25 Alain Magloire
 	Added the PathEntry in the AllCoreTests suite
 
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScannerTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScannerTest.java
index d86b5386c33..eafa711e922 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScannerTest.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScannerTest.java
@@ -85,8 +85,8 @@ public class BaseScannerTest extends TestCase {
 	{
 		try {
 			IToken t= scanner.nextToken();
-			assertTrue(t.getType() == IToken.tIDENTIFIER);
-			assertTrue(t.getImage().equals(expectedImage));
+			assertEquals( t.getType(), IToken.tIDENTIFIER );
+			assertEquals(t.getImage(), expectedImage );
 		} catch (EndOfFileException e) {
 			assertTrue(false);
 		} 
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java
index 511dc7aa640..ef6ef72258c 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java
@@ -1555,4 +1555,13 @@ public class ScannerTestCase extends BaseScannerTest
     	scanner.nextToken(); 
     	assertEquals( callback.problems.size(), 1 );
     }
+    
+    public void testGerman() throws ScannerException
+	{
+    	initializeScanner("\"Lißä\" 'ß' /* Lißä */ Lißä ");
+    	validateString("Lißä");
+    	validateChar( 'ß' );
+    	validateIdentifier( "Lißä");
+    	validateEOF();
+    }
 }
diff --git a/core/org.eclipse.cdt.core/parser/ChangeLog-parser b/core/org.eclipse.cdt.core/parser/ChangeLog-parser
index 284dc976b8e..4bc2de2e4f9 100644
--- a/core/org.eclipse.cdt.core/parser/ChangeLog-parser
+++ b/core/org.eclipse.cdt.core/parser/ChangeLog-parser
@@ -1,3 +1,6 @@
+2004-02-25 John Camelon
+	Updated Scanner to allow for invalid identifier names despite C++'s best efforts at maintaining its honour.  
+
 2004-02-24 John Camelon
 	Refactoring Scanner.handleInclusion to be more modular.
 
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java
index a5b4fcfefa4..b5ad423a7ee 100644
--- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java
@@ -994,8 +994,12 @@ public class Scanner implements IScanner {
 				
 		
 			} else if (
-				((c >= 'a') && (c <= 'z'))
-					|| ((c >= 'A') && (c <= 'Z')) || (c == '_')) {
+					// JOHNC - Accept non-ASCII Input
+//				((c >= 'a') && (c <= 'z'))
+//					|| ((c >= 'A') && (c <= 'Z')) || (c == '_')) {
+					Character.isLetter((char)c) || ( c == '_') 
+			)
+			{
                         
                 int baseOffset = lastContext.getOffset() - lastContext.undoStackSize() - 1;
 						
@@ -1005,10 +1009,13 @@ public class Scanner implements IScanner {
 
 				c = getChar();				
 				
-				while (((c >= 'a') && (c <= 'z'))
-					|| ((c >= 'A') && (c <= 'Z'))
-					|| ((c >= '0') && (c <= '9'))
-					|| (c == '_')) {
+				while (
+						Character.isUnicodeIdentifierPart( (char)c)
+//					((c >= 'a') && (c <= 'z'))
+//					|| ((c >= 'A') && (c <= 'Z'))
+//					|| ((c >= '0') && (c <= '9'))
+//					|| (c == '_')
+					) {
 					buff.append((char) c);
 					c = getChar();
 				}
@@ -2003,7 +2010,6 @@ public class Scanner implements IScanner {
                 if (tokenImage.length() > 0) throw endOfMacroToken;                
                 c = getChar();
                 continue;
-                
             } else if (c == '"') {
        
                 if (tokenImage.length() > 0) throw endOfMacroToken;