From b56089841337bade8895b41dc0d7dd5e0c826982 Mon Sep 17 00:00:00 2001 From: Doug Schaefer Date: Mon, 14 Jun 2004 16:04:28 +0000 Subject: [PATCH] Checking in Scanner2 and friends. This should hopefully be a faster scanner and set us up for using char[]'s up the stack. --- .../core/parser/tests/BaseScanner2Test.java | 247 +++ .../core/parser/tests/CharArrayUtilsTest.java | 29 +- .../cdt/core/parser/tests/Scanner2Test.java | 1614 +++++++++++++++++ .../eclipse/cdt/core/parser/CharArrayMap.java | 101 -- .../org/eclipse/cdt/core/parser/IToken.java | 5 +- .../core/parser/scanner/ScannerUtility.java | 8 +- .../core/parser/scanner2/CharArrayIntMap.java | 50 + .../core/parser/scanner2/CharArrayMap.java | 137 ++ .../parser/scanner2/CharArrayObjectMap.java | 54 + .../core/parser/scanner2}/CharArrayPool.java | 2 +- .../core/parser/scanner2}/CharArrayUtils.java | 25 +- .../parser/scanner2/FunctionStyleMacro.java | 31 + .../parser/scanner2/MacroExpansionToken.java | 134 ++ .../parser/scanner2/ObjectStyleMacro.java | 25 + .../core/parser/scanner2/Scanner2.java | 1535 ++++++++++++++++ 15 files changed, 3872 insertions(+), 125 deletions(-) create mode 100644 core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScanner2Test.java create mode 100644 core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/Scanner2Test.java delete mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CharArrayMap.java create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayIntMap.java create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayMap.java create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayObjectMap.java rename core/org.eclipse.cdt.core/parser/org/eclipse/cdt/{core/parser => internal/core/parser/scanner2}/CharArrayPool.java (98%) rename core/org.eclipse.cdt.core/parser/org/eclipse/cdt/{core/parser => internal/core/parser/scanner2}/CharArrayUtils.java (63%) create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/FunctionStyleMacro.java create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/MacroExpansionToken.java create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/ObjectStyleMacro.java create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScanner2Test.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScanner2Test.java new file mode 100644 index 00000000000..7e62deabd5c --- /dev/null +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/BaseScanner2Test.java @@ -0,0 +1,247 @@ +/******************************************************************************* + * Copyright (c) 2001 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Corp. - Rational Software - initial implementation + ******************************************************************************/ + +package org.eclipse.cdt.core.parser.tests; + +import java.util.List; + +import junit.framework.TestCase; + +import org.eclipse.cdt.core.parser.CodeReader; +import org.eclipse.cdt.core.parser.EndOfFileException; +import org.eclipse.cdt.core.parser.IParserLogService; +import org.eclipse.cdt.core.parser.IScannerInfo; +import org.eclipse.cdt.core.parser.ISourceElementRequestor; +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.NullSourceElementRequestor; +import org.eclipse.cdt.core.parser.ParserFactory; +import org.eclipse.cdt.core.parser.ParserFactoryError; +import org.eclipse.cdt.core.parser.ParserLanguage; +import org.eclipse.cdt.core.parser.ParserMode; +import org.eclipse.cdt.core.parser.ScannerException; +import org.eclipse.cdt.core.parser.ScannerInfo; +import org.eclipse.cdt.core.parser.extension.ExtensionDialect; +import org.eclipse.cdt.internal.core.parser.ParserExtensionFactory; +import org.eclipse.cdt.internal.core.parser.scanner2.CharArrayUtils; +import org.eclipse.cdt.internal.core.parser.scanner2.ObjectStyleMacro; +import org.eclipse.cdt.internal.core.parser.scanner2.Scanner2; + +/** + * @author jcamelon + * + */ +public class BaseScanner2Test extends TestCase { + + protected Scanner2 scanner; + + public BaseScanner2Test( String x ) + { + super(x); + } + + protected void initializeScanner( String input, ParserMode mode ) throws ParserFactoryError + { + initializeScanner( input, mode, new NullSourceElementRequestor( mode )); + } + + protected void initializeScanner( String input, ParserMode mode, ISourceElementRequestor requestor ) throws ParserFactoryError + { + scanner = createScanner( new CodeReader(input.toCharArray()), new ScannerInfo(), mode, ParserLanguage.CPP, requestor, null, null ); //$NON-NLS-1$ + } + + protected void initializeScanner(String input) throws ParserFactoryError + { + initializeScanner( input, ParserMode.COMPLETE_PARSE ); + } + + public static Scanner2 createScanner( CodeReader code, IScannerInfo config, ParserMode mode, ParserLanguage language, ISourceElementRequestor requestor, IParserLogService log, List workingCopies ) throws ParserFactoryError + { + if( config == null ) throw new ParserFactoryError( ParserFactoryError.Kind.NULL_CONFIG ); + if( language == null ) throw new ParserFactoryError( ParserFactoryError.Kind.NULL_LANGUAGE ); + IParserLogService logService = ( log == null ) ? ParserFactory.createDefaultLogService() : log; + ParserMode ourMode = ( (mode == null )? ParserMode.COMPLETE_PARSE : mode ); + ISourceElementRequestor ourRequestor = (( requestor == null) ? new NullSourceElementRequestor() : requestor ); + return new Scanner2( code, config, ourRequestor, ourMode, language, logService, new ParserExtensionFactory( ExtensionDialect.GCC ).createScannerExtension(), workingCopies ); + } + + public int fullyTokenize() throws Exception + { + try + { + IToken t= scanner.nextToken(); + while (t != null) + { + if (verbose) + System.out.println("Token t = " + t); //$NON-NLS-1$ + + if ((t.getType()> IToken.tLAST)) + System.out.println("Unknown type for token " + t); //$NON-NLS-1$ + t= scanner.nextToken(); + } + } + catch ( EndOfFileException e) + { + } + catch (ScannerException se) + { + throw se; + } + return scanner.getCount(); + } + public void validateIdentifier(String expectedImage) throws ScannerException + { + try { + IToken t= scanner.nextToken(); + assertEquals( t.getType(), IToken.tIDENTIFIER ); + assertEquals(t.getImage(), expectedImage ); + } catch (EndOfFileException e) { + assertTrue(false); + } + } + + public void validateInteger(String expectedImage) throws ScannerException + { + try { + IToken t= scanner.nextToken(); + assertTrue(t.getType() == IToken.tINTEGER); + assertTrue(t.getImage().equals(expectedImage)); + } catch (EndOfFileException e) { + assertTrue(false); + } + } + + public void validateFloatingPointLiteral(String expectedImage) throws ScannerException + { + try { + IToken t= scanner.nextToken(); + assertTrue(t.getType() == IToken.tFLOATINGPT); + assertTrue(t.getImage().equals(expectedImage)); + } catch (EndOfFileException e) { + assertTrue(false); + } + } + + public void validateChar( char expected )throws ScannerException + { + try { + IToken t= scanner.nextToken(); + assertTrue(t.getType() == IToken.tCHAR ); + Character c = new Character( expected ); + assertEquals( t.getImage(), c.toString() ); + } catch (EndOfFileException e) { + assertTrue(false); + } + } + + public void validateChar( String expected ) throws ScannerException + { + try { + IToken t= scanner.nextToken(); + assertTrue(t.getType() == IToken.tCHAR ); + assertEquals( t.getImage(), expected ); + } catch (EndOfFileException e) { + assertTrue(false); + } + } + + public void validateString( String expectedImage ) throws ScannerException + { + validateString( expectedImage, false ); + } + + public void validateString(String expectedImage, boolean lString ) throws ScannerException + { + try { + IToken t= scanner.nextToken(); + if( lString ) + assertTrue(t.getType() == IToken.tLSTRING); + else + assertTrue(t.getType() == IToken.tSTRING); + assertTrue(t.getImage().equals(expectedImage)); + } catch (EndOfFileException e) { + assertTrue(false); + } + } + + public void validateToken(int tokenType) throws ScannerException + { + try { + IToken t= scanner.nextToken(); + assertTrue(t.getType() == tokenType); + } catch (EndOfFileException e) { + assertTrue(false); + } + } + + public void validateBalance(int expected) + { + assertTrue(scanner.getDepth() == expected); + } + + public void validateBalance() + { + assertTrue(scanner.getDepth() == 0); + } + + public void validateEOF() throws ScannerException + { + try { + assertNull(scanner.nextToken()); + } catch (EndOfFileException e) { + } + } + + public void validateDefinition(String name, String value) + { + Object expObject = scanner.getRealDefinitions().get(name.toCharArray()); + assertNotNull(expObject); + assertTrue(expObject instanceof ObjectStyleMacro); + assertTrue(CharArrayUtils.equals(value.toCharArray(), ((ObjectStyleMacro)expObject).expansion)); + } + + public void validateDefinition(String name, int value) + { + String definition= null; + definition= scanner.getDefinition(name).getExpansionSignature(); + assertNotNull(definition); + int intValue= (Integer.valueOf(definition)).intValue(); + assertEquals(value, intValue); + } + + public void validateAsUndefined(String name) + { + assertNull(scanner.getDefinition(name)); + } + + public static final String EXCEPTION_THROWN = "Exception thrown "; //$NON-NLS-1$ + + public static final String EXPECTED_FAILURE = "This statement should not be reached " //$NON-NLS-1$ + + "as we sent in bad preprocessor input to the scanner"; //$NON-NLS-1$ + + public static final boolean verbose = false; + + + /** + * @param string + */ + protected void validateWideChar(String string) throws Exception + { + try { + IToken t= scanner.nextToken(); + assertTrue(t.getType() == IToken.tLCHAR ); + assertEquals( t.getImage(), string ); + } catch (EndOfFileException e) { + assertTrue(false); + } + } + + +} diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/CharArrayUtilsTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/CharArrayUtilsTest.java index 6d7e44a0de7..c617d3375f3 100644 --- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/CharArrayUtilsTest.java +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/CharArrayUtilsTest.java @@ -1,22 +1,23 @@ -/* - * Created on May 28, 2004 - * - * TODO To change the template for this generated file go to - * Window - Preferences - Java - Code Style - Code Templates - */ +/******************************************************************************* + * Copyright (c) 2004 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Corporation - initial implementation + ******************************************************************************/ package org.eclipse.cdt.core.parser.tests; import junit.framework.TestCase; -import org.eclipse.cdt.core.parser.CharArrayMap; -import org.eclipse.cdt.core.parser.CharArrayPool; -import org.eclipse.cdt.core.parser.CharArrayUtils; +import org.eclipse.cdt.internal.core.parser.scanner2.CharArrayObjectMap; +import org.eclipse.cdt.internal.core.parser.scanner2.CharArrayPool; +import org.eclipse.cdt.internal.core.parser.scanner2.CharArrayUtils; /** - * @author dschaefe - * - * TODO To change the template for this generated type comment go to - * Window - Preferences - Java - Code Style - Code Templates + * @author Doug Schaefer */ public class CharArrayUtilsTest extends TestCase { @@ -75,7 +76,7 @@ public class CharArrayUtilsTest extends TestCase { } public void testMapAdd() { - CharArrayMap map = new CharArrayMap(4); + CharArrayObjectMap map = new CharArrayObjectMap(4); char[] key1 = "key1".toCharArray(); Object value1 = new Integer(43); map.put(key1, value1); diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/Scanner2Test.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/Scanner2Test.java new file mode 100644 index 00000000000..408d8937d2e --- /dev/null +++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/Scanner2Test.java @@ -0,0 +1,1614 @@ +/******************************************************************************* + * Copyright (c) 2003,2004 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Corp. - Rational Software - initial implementation + ******************************************************************************/ + +package org.eclipse.cdt.core.parser.tests; + +import java.io.StringWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.cdt.core.parser.IGCCToken; +import org.eclipse.cdt.core.parser.IMacroDescriptor; +import org.eclipse.cdt.core.parser.IProblem; +import org.eclipse.cdt.core.parser.ISourceElementRequestor; +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.NullSourceElementRequestor; +import org.eclipse.cdt.core.parser.ParserFactoryError; +import org.eclipse.cdt.core.parser.ParserMode; +import org.eclipse.cdt.core.parser.ScannerException; +import org.eclipse.cdt.core.parser.ast.IASTInclusion; + +/** + * @author jcamelon + */ +public class Scanner2Test extends BaseScanner2Test +{ + public class TableRow + { + private int[] values; + private int length; + + public TableRow(int[] v) + { + length= v.length; + values= new int[length]; + System.arraycopy(v, 0, values, 0, length); + } + + public String toString() + { + StringBuffer s= new StringBuffer(); + for (int i= 0; i < length; ++i) + { + s.append("var").append(i).append("=").append(values[i]).append(" "); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } + return s.toString(); + } + + public String symbolName(int index) + { + return "DEFINITION" + index; //$NON-NLS-1$ + } + + public int symbolValue(int index) + { + return new Long(Math.round(Math.pow(index, index))).intValue(); + } + + public String generateCode() + { + if (length < 2) + { + return "Array must have at least 2 elements"; //$NON-NLS-1$ + } + int numberOfElsifs= length - 1; + StringBuffer buffer= new StringBuffer(); + buffer.append("#if ").append(values[0]).append("\n#\tdefine "); //$NON-NLS-1$ //$NON-NLS-2$ + buffer.append(symbolName(0)).append(" ").append(symbolValue(0)); //$NON-NLS-1$ + for (int i= 0; i < numberOfElsifs; ++i) + buffer + .append("\n#elif ") //$NON-NLS-1$ + .append(values[1 + i]) + .append("\n#\tdefine ") //$NON-NLS-1$ + .append(symbolName(i + 1)) + .append(" ") //$NON-NLS-1$ + .append(symbolValue(i + 1)); + buffer + .append("\n#else \n#\tdefine ") //$NON-NLS-1$ + .append(symbolName(length)) + .append(" ") //$NON-NLS-1$ + .append(symbolValue(length)) + .append("\n#endif"); //$NON-NLS-1$ + return buffer.toString(); + } + + public int selectWinner() + { + for (int i= 0; i < values.length; ++i) + { + if (values[i] != 0) + { + return i; + } + } + return length; + } + /** + * Returns the length. + * @return int + */ + public int getLength() + { + return length; + } + + } + + public class TruthTable + { + private int numberOfVariables; + private int numberOfRows; + public TableRow[] rows; + + public TruthTable(int n) + { + numberOfVariables= n; + numberOfRows= new Long(Math.round(Math.pow(2, n))).intValue(); + + rows= new TableRow[numberOfRows]; + for (int i= 0; i < numberOfRows; ++i) + { + String Z= Integer.toBinaryString(i); + + int[] input= new int[numberOfVariables]; + for (int j= 0; j < numberOfVariables; ++j) + { + int padding= numberOfVariables - Z.length(); + int k= 0; + for (; k < padding; ++k) + { + input[k]= 0; + } + for (int l= 0; l < Z.length(); ++l) + { + char c= Z.charAt(l); + int value= Character.digit(c, 10); + input[k++]= value; + } + } + rows[i]= new TableRow(input); + } + } + /** + * Returns the numberOfRows. + * @return int + */ + public int getNumberOfRows() + { + return numberOfRows; + } + + } + + public final static int SIZEOF_TRUTHTABLE = 10; + + + public void testWeirdStrings() throws Exception + { + try + { + initializeScanner( "Living Life L\"LONG\""); //$NON-NLS-1$ + validateIdentifier( "Living" ); //$NON-NLS-1$ + validateIdentifier( "Life" ); //$NON-NLS-1$ + validateString("LONG", true); //$NON-NLS-1$ + validateEOF(); + } + catch( ScannerException se ) + { + fail(EXCEPTION_THROWN + se.toString()); + } + + } + + + public void testNumerics()throws Exception + { + try + { + initializeScanner("3.0 0.9 .5 3. 4E5 2.01E-03 ..."); //$NON-NLS-1$ + validateFloatingPointLiteral( "3.0"); //$NON-NLS-1$ + validateFloatingPointLiteral( "0.9"); //$NON-NLS-1$ + validateFloatingPointLiteral( ".5"); //$NON-NLS-1$ + validateFloatingPointLiteral( "3."); //$NON-NLS-1$ + validateFloatingPointLiteral( "4E5"); //$NON-NLS-1$ + validateFloatingPointLiteral( "2.01E-03" ); //$NON-NLS-1$ + validateToken( IToken.tELLIPSIS ); + validateEOF(); + } + catch( ScannerException se ) + { + fail(EXCEPTION_THROWN + se.toString()); + } + + } + + + /** + * Constructor for ScannerTestCase. + * @param name + */ + public Scanner2Test(String name) + { + super(name); + } + + public void testPreprocessorDefines()throws Exception + { + initializeScanner("#define SIMPLE_NUMERIC 5\nint x = SIMPLE_NUMERIC"); //$NON-NLS-1$ + validateToken(IToken.t_int); + validateDefinition("SIMPLE_NUMERIC", "5"); //$NON-NLS-1$ //$NON-NLS-2$ + validateIdentifier("x"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateInteger("5"); //$NON-NLS-1$ + validateEOF(); + + initializeScanner("#define SIMPLE_STRING \"This is a simple string.\"\n\nconst char * myVariable = SIMPLE_STRING;"); //$NON-NLS-1$ + validateToken(IToken.t_const); + validateDefinition("SIMPLE_STRING", "\"This is a simple string.\""); //$NON-NLS-1$ //$NON-NLS-2$ + validateToken(IToken.t_char); + validateToken(IToken.tSTAR); + validateIdentifier("myVariable"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateString("This is a simple string."); //$NON-NLS-1$ + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define FOOL 5 \n int tryAFOOL = FOOL + FOOL;"); //$NON-NLS-1$ + validateToken(IToken.t_int); + validateIdentifier("tryAFOOL"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateInteger("5"); //$NON-NLS-1$ + validateToken(IToken.tPLUS); + validateInteger("5"); //$NON-NLS-1$ + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define FOOL 5 \n int FOOLer = FOOL;"); //$NON-NLS-1$ + validateToken(IToken.t_int); + validateIdentifier("FOOLer"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateInteger("5"); //$NON-NLS-1$ + validateToken(IToken.tSEMI); + validateEOF(); + + // the case we were failing against in ctype.h + // this is a definition, not a macro! + initializeScanner("#define _ALPHA (0x0100|_UPPER|_LOWER)"); //$NON-NLS-1$ + validateEOF(); + validateDefinition("_ALPHA", "(0x0100|_UPPER|_LOWER)"); //$NON-NLS-1$ //$NON-NLS-2$ + + // test for comments after the macro + initializeScanner("#define NO_COMMENT// ignore me"); //$NON-NLS-1$ + validateEOF(); + validateDefinition("NO_COMMENT", ""); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#define NO_COMMENT/* ignore me*/"); //$NON-NLS-1$ + validateEOF(); + validateDefinition("NO_COMMENT", ""); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#define ANSWER 42 // i think"); //$NON-NLS-1$ + validateEOF(); + validateDefinition("ANSWER", "42"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#define ANSWER 42 /* i think */"); //$NON-NLS-1$ + validateEOF(); + validateDefinition("ANSWER", "42"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#define MULTILINE 3 /* comment \n that goes more than one line */"); //$NON-NLS-1$ + validateEOF(); + validateDefinition("MULTILINE", "3"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#define MULTICOMMENT X /* comment1 */ + Y /* comment 2 */"); //$NON-NLS-1$ + validateEOF(); + validateDefinition("MULTICOMMENT", "X /* comment1 */ + Y"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#define SIMPLE_STRING This is a simple string.\n"); //$NON-NLS-1$ + validateEOF(); + validateDefinition( + "SIMPLE_STRING", //$NON-NLS-1$ + "This is a simple string."); //$NON-NLS-1$ + + initializeScanner("# define SIMPLE_NUMERIC 5\n"); //$NON-NLS-1$ + validateEOF(); + validateDefinition("SIMPLE_NUMERIC", "5"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("# define SIMPLE_NUMERIC 5\n"); //$NON-NLS-1$ + validateEOF(); + validateDefinition("SIMPLE_NUMERIC", "5"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#define SIMPLE_STRING \"This is a simple string.\"\n"); //$NON-NLS-1$ + validateEOF(); + validateDefinition( + "SIMPLE_STRING", //$NON-NLS-1$ + "\"This is a simple string.\""); //$NON-NLS-1$ + + initializeScanner("#define SIMPLE_STRING This is a simple string.\n"); //$NON-NLS-1$ + validateEOF(); + validateDefinition( + "SIMPLE_STRING", //$NON-NLS-1$ + "This is a simple string."); //$NON-NLS-1$ + + initializeScanner("#define FLAKE\n\nFLAKE"); //$NON-NLS-1$ + validateEOF(); + validateDefinition("FLAKE", ""); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#define SIMPLE_STRING This is a simple string.\\\n Continue please."); //$NON-NLS-1$ + validateEOF(); + validateDefinition( + "SIMPLE_STRING", //$NON-NLS-1$ + "This is a simple string.\\\n Continue please."); //$NON-NLS-1$ + } + + + public void testConcatenation() + { + try + { + initializeScanner("#define F1 3\n#define F2 F1##F1\nint x=F2;"); //$NON-NLS-1$ + validateToken(IToken.t_int); + validateDefinition("F1", "3"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition( "F2", "F1##F1"); //$NON-NLS-1$ //$NON-NLS-2$ + validateIdentifier("x"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateInteger("33"); //$NON-NLS-1$ + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define PREFIX RT_\n#define RUN PREFIX##Run"); //$NON-NLS-1$ + validateEOF(); + validateDefinition( "PREFIX", "RT_" ); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition( "RUN", "PREFIX##Run" ); //$NON-NLS-1$ //$NON-NLS-2$ + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + + try + { + initializeScanner( "#define DECLARE_HANDLE(name) struct name##__ { int unused; }; typedef struct name##__ *name\n DECLARE_HANDLE( joe )" ); //$NON-NLS-1$ + validateToken( IToken.t_struct ); + validateIdentifier( "joe__"); //$NON-NLS-1$ + validateToken( IToken.tLBRACE); + validateToken( IToken.t_int ); + validateIdentifier( "unused"); //$NON-NLS-1$ + validateToken( IToken.tSEMI ); + validateToken( IToken.tRBRACE ); + validateToken( IToken.tSEMI ); + validateToken( IToken.t_typedef ); + validateToken( IToken.t_struct ); + validateIdentifier( "joe__" ); //$NON-NLS-1$ + validateToken( IToken.tSTAR ); + validateIdentifier( "joe"); //$NON-NLS-1$ + validateEOF(); + } + catch( Exception e ) + { + fail(EXCEPTION_THROWN + e.toString()); + } + } + + public void testSimpleIfdef() + { + try + { + + initializeScanner("#define SYMBOL 5\n#ifdef SYMBOL\nint counter(SYMBOL);\n#endif"); //$NON-NLS-1$ + + validateToken(IToken.t_int); + validateIdentifier("counter"); //$NON-NLS-1$ + validateToken(IToken.tLPAREN); + validateInteger("5"); //$NON-NLS-1$ + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define SYMBOL 5\n#ifndef SYMBOL\nint counter(SYMBOL);\n#endif"); //$NON-NLS-1$ + validateEOF(); + + initializeScanner("#ifndef DEFINED\n#define DEFINED 100\n#endif\nint count = DEFINED;"); //$NON-NLS-1$ + validateToken(IToken.t_int); + validateDefinition("DEFINED", "100"); //$NON-NLS-1$ //$NON-NLS-2$ + + validateIdentifier("count"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateInteger("100"); //$NON-NLS-1$ + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#ifndef DEFINED\n#define DEFINED 100\n#endif\nint count = DEFINED;"); //$NON-NLS-1$ + scanner.addDefinition("DEFINED", "101"); //$NON-NLS-1$ //$NON-NLS-2$ + + validateDefinition("DEFINED", "101"); //$NON-NLS-1$ //$NON-NLS-2$ + validateToken(IToken.t_int); + validateIdentifier("count"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateInteger("101"); //$NON-NLS-1$ + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner( "/* NB: This is #if 0'd out */"); //$NON-NLS-1$ + validateEOF(); + + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + } + + public void testMultipleLines() throws Exception + { + Writer code = new StringWriter(); + code.write( "#define COMPLEX_MACRO 33 \\\n"); //$NON-NLS-1$ + code.write( " + 44\n\nCOMPLEX_MACRO"); //$NON-NLS-1$ + initializeScanner( code.toString() ); + validateInteger( "33" ); //$NON-NLS-1$ + validateToken( IToken.tPLUS ); + validateInteger( "44" ); //$NON-NLS-1$ + } + + public void testSlightlyComplexIfdefStructure() + { + try + { + initializeScanner("#ifndef BASE\n#define BASE 10\n#endif\n#ifndef BASE\n#error BASE is defined\n#endif"); //$NON-NLS-1$ + validateEOF(); + validateBalance(); + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + + try + { + initializeScanner("#ifndef ONE\n#define ONE 1\n#ifdef TWO\n#define THREE ONE + TWO\n#endif\n#endif\nint three(THREE);"); //$NON-NLS-1$ + + validateToken(IToken.t_int); + validateDefinition("ONE", "1"); //$NON-NLS-1$ //$NON-NLS-2$ + validateAsUndefined("TWO"); //$NON-NLS-1$ + validateAsUndefined("THREE"); //$NON-NLS-1$ + validateIdentifier("three"); //$NON-NLS-1$ + validateToken(IToken.tLPAREN); + validateIdentifier("THREE"); //$NON-NLS-1$ + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + validateBalance(); + + initializeScanner("#ifndef ONE\n#define ONE 1\n#ifdef TWO\n#define THREE ONE + TWO\n#endif\n#endif\nint three(THREE);"); //$NON-NLS-1$ + scanner.addDefinition("TWO", "2"); //$NON-NLS-1$ //$NON-NLS-2$ + validateToken(IToken.t_int); + validateDefinition("ONE", "1"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("TWO", "2"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("THREE", "ONE + TWO"); //$NON-NLS-1$ //$NON-NLS-2$ + + validateIdentifier("three"); //$NON-NLS-1$ + validateToken(IToken.tLPAREN); + validateInteger("1"); //$NON-NLS-1$ + validateToken(IToken.tPLUS); + validateInteger("2"); //$NON-NLS-1$ + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + validateBalance(); + + initializeScanner("#ifndef FOO\n#define FOO 4\n#else\n#undef FOO\n#define FOO 6\n#endif"); //$NON-NLS-1$ + validateEOF(); + validateBalance(); + validateDefinition("FOO", "4"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#ifndef FOO\n#define FOO 4\n#else\n#undef FOO\n#define FOO 6\n#endif"); //$NON-NLS-1$ + scanner.addDefinition("FOO", "2"); //$NON-NLS-1$ //$NON-NLS-2$ + validateEOF(); + validateBalance(); + validateDefinition("FOO", "6"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#ifndef ONE\n# define ONE 1\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#else\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#endif\n"); //$NON-NLS-1$ + validateEOF(); + validateBalance(); + validateDefinition("ONE", "1"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("TWO", "ONE + ONE"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#ifndef ONE\n# define ONE 1\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#else\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#endif\n"); //$NON-NLS-1$ + scanner.addDefinition("ONE", "one"); //$NON-NLS-1$ //$NON-NLS-2$ + validateEOF(); + validateBalance(); + validateDefinition("ONE", "one"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("TWO", "ONE + ONE"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#ifndef ONE\n# define ONE 1\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#else\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#endif\n"); //$NON-NLS-1$ + scanner.addDefinition("ONE", "one"); //$NON-NLS-1$ //$NON-NLS-2$ + scanner.addDefinition("TWO", "two"); //$NON-NLS-1$ //$NON-NLS-2$ + validateEOF(); + validateBalance(); + + validateDefinition("ONE", "one"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("TWO", "2"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#ifndef ONE\n# define ONE 1\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#else\n# ifndef TWO\n# define TWO ONE + ONE \n# else\n# undef TWO\n# define TWO 2 \n# endif\n#endif\n"); //$NON-NLS-1$ + scanner.addDefinition("TWO", "two"); //$NON-NLS-1$ //$NON-NLS-2$ + validateEOF(); + validateBalance(); + + validateDefinition("ONE", "1"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("TWO", "2"); //$NON-NLS-1$ //$NON-NLS-2$ + + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + } + + public void testIfs() + { + try + { + initializeScanner("#if 0\n#error NEVER\n#endif\n"); //$NON-NLS-1$ + validateEOF(); + validateBalance(); + + initializeScanner("#define X 5\n#define Y 7\n#if (X < Y)\n#define Z X + Y\n#endif"); //$NON-NLS-1$ + validateEOF(); + validateBalance(); + validateDefinition("X", "5"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("Y", "7"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("Z", "X + Y"); //$NON-NLS-1$ //$NON-NLS-2$ + + initializeScanner("#if T < 20\n#define Z T + 1\n#endif"); //$NON-NLS-1$ + scanner.addDefinition("X", "5"); //$NON-NLS-1$ //$NON-NLS-2$ + scanner.addDefinition("Y", "7"); //$NON-NLS-1$ //$NON-NLS-2$ + scanner.addDefinition("T", "X + Y"); //$NON-NLS-1$ //$NON-NLS-2$ + validateEOF(); + validateBalance(); + validateDefinition("X", "5"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("Y", "7"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("T", "X + Y"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("Z", "T + 1"); //$NON-NLS-1$ //$NON-NLS-2$ + + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + + try + { + initializeScanner("#if ( 10 / 5 ) != 2\n#error 10/5 seems to not equal 2 anymore\n#endif\n"); //$NON-NLS-1$ + validateEOF(); + validateBalance(); + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + + try + { + initializeScanner("#ifndef FIVE \n#define FIVE 5\n#endif \n#ifndef TEN\n#define TEN 2 * FIVE\n#endif\n#if TEN != 10\n#define MISTAKE 1\n#error Five does not equal 10\n#endif\n"); //$NON-NLS-1$ + scanner.addDefinition("FIVE", "55"); //$NON-NLS-1$ //$NON-NLS-2$ + validateEOF(); + fail(EXPECTED_FAILURE); + } + catch (ScannerException se) + { + validateBalance(1); + validateDefinition("FIVE", "55"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("TEN", "2 * FIVE"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("MISTAKE", "1"); //$NON-NLS-1$ //$NON-NLS-2$ + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + + try + { + initializeScanner("#if ((( FOUR / TWO ) * THREE )< FIVE )\n#error 6 is not less than 5 \n#endif\n#if ( ( FIVE * ONE ) != (( (FOUR) + ONE ) * ONE ) )\n#error 5 should equal 5\n#endif \n"); //$NON-NLS-1$ + + scanner.addDefinition("ONE", "1"); //$NON-NLS-1$ //$NON-NLS-2$ + scanner.addDefinition("TWO", "(ONE + ONE)"); //$NON-NLS-1$ //$NON-NLS-2$ + scanner.addDefinition("THREE", "(TWO + ONE)"); //$NON-NLS-1$ //$NON-NLS-2$ + scanner.addDefinition("FOUR", "(TWO * TWO)"); //$NON-NLS-1$ //$NON-NLS-2$ + scanner.addDefinition("FIVE", "(THREE + TWO)"); //$NON-NLS-1$ //$NON-NLS-2$ + + validateEOF(); + validateBalance(); + validateDefinition("ONE", "1"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("TWO", "(ONE + ONE)"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("THREE", "(TWO + ONE)"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("FOUR", "(TWO * TWO)"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("FIVE", "(THREE + TWO)"); //$NON-NLS-1$ //$NON-NLS-2$ + + TruthTable table= new TruthTable(SIZEOF_TRUTHTABLE); + int numberOfRows= table.getNumberOfRows(); + TableRow[] rows= table.rows; + + for (int i= 0; i < numberOfRows; ++i) + { + TableRow row= rows[i]; + String code= row.generateCode(); + if (verbose) + System.out.println("\n\nRow " + i + " has code\n" + code); //$NON-NLS-1$ //$NON-NLS-2$ + initializeScanner(code); + validateEOF(); + validateBalance(); + validateAllDefinitions(row); + } + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + + try + { + initializeScanner("#if ! 0\n#error Correct!\n#endif"); //$NON-NLS-1$ + scanner.nextToken(); + fail(EXPECTED_FAILURE); + } + catch (ScannerException se) + { + validateBalance(1); + assertEquals( se.getProblem().getID(), IProblem.PREPROCESSOR_POUND_ERROR); + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + } + + public void testPreprocessorMacros() + { + try + { + initializeScanner("#define GO(x) x+1\nint y(5);\ny = GO(y);"); //$NON-NLS-1$ + validateToken(IToken.t_int); + validateIdentifier("y"); //$NON-NLS-1$ + validateToken(IToken.tLPAREN); + validateInteger("5"); //$NON-NLS-1$ + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + + IMacroDescriptor descriptor= + scanner.getDefinition("GO"); //$NON-NLS-1$ + String [] parms= descriptor.getParameters(); + assertNotNull(parms); + assertTrue(parms.length == 1); + String parm1= parms[0]; + assertTrue(parm1.equals("x")); //$NON-NLS-1$ + IToken [] expansion= descriptor.getTokenizedExpansion(); + assertNotNull(parms); + assertTrue(expansion.length == 3); + assertTrue((expansion[0]).getType() == IToken.tIDENTIFIER); + assertTrue((expansion[0]).getImage().equals("x")); //$NON-NLS-1$ + assertTrue((expansion[1]).getType() == IToken.tPLUS); + assertTrue((expansion[2]).getType() == IToken.tINTEGER); + assertTrue((expansion[2]).getImage().equals("1")); //$NON-NLS-1$ + + validateIdentifier("y"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateIdentifier("y"); //$NON-NLS-1$ + validateToken(IToken.tPLUS); + validateInteger("1"); //$NON-NLS-1$ + validateToken(IToken.tSEMI); + validateEOF(); + validateBalance(); + + initializeScanner( + "#define ONE 1\n" //$NON-NLS-1$ + + "#define SUM(a,b,c,d,e,f,g) ( a + b + c + d + e + f + g )\n" //$NON-NLS-1$ + + "int daSum = SUM(ONE,3,5,7,9,11,13);"); //$NON-NLS-1$ + validateToken(IToken.t_int); + validateIdentifier("daSum"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateToken(IToken.tLPAREN); + validateInteger("1"); //$NON-NLS-1$ + validateToken(IToken.tPLUS); + validateInteger("3"); //$NON-NLS-1$ + validateToken(IToken.tPLUS); + validateInteger("5"); //$NON-NLS-1$ + validateToken(IToken.tPLUS); + validateInteger("7"); //$NON-NLS-1$ + validateToken(IToken.tPLUS); + validateInteger("9"); //$NON-NLS-1$ + validateToken(IToken.tPLUS); + validateInteger("11"); //$NON-NLS-1$ + validateToken(IToken.tPLUS); + validateInteger("13"); //$NON-NLS-1$ + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + + IMacroDescriptor macro= scanner.getDefinition("SUM"); //$NON-NLS-1$ + String [] params= macro.getParameters(); + assertNotNull(params); + assertTrue(params.length == 7); + + IToken [] tokens= macro.getTokenizedExpansion(); + assertNotNull(tokens); + assertTrue(tokens.length == 15); + + initializeScanner("#define LOG( format, var1) printf( format, var1 )\nLOG( \"My name is %s\", \"Bogdan\" );\n"); //$NON-NLS-1$ + validateIdentifier("printf"); //$NON-NLS-1$ + validateToken(IToken.tLPAREN); + validateString("My name is %s"); //$NON-NLS-1$ + validateToken(IToken.tCOMMA); + validateString("Bogdan"); //$NON-NLS-1$ + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define INCR( x ) ++x\nint y(2);\nINCR(y);"); //$NON-NLS-1$ + validateToken(IToken.t_int); + validateIdentifier("y"); //$NON-NLS-1$ + validateToken(IToken.tLPAREN); + validateInteger("2"); //$NON-NLS-1$ + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateToken(IToken.tINCR); + validateIdentifier("y"); //$NON-NLS-1$ + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define CHECK_AND_SET( x, y, z ) if( x ) { \\\n y = z; \\\n }\n\nCHECK_AND_SET( 1, balance, 5000 );\nCHECK_AND_SET( confused(), you, dumb );"); //$NON-NLS-1$ + validateToken(IToken.t_if); + validateToken(IToken.tLPAREN); + validateInteger("1"); //$NON-NLS-1$ + validateToken(IToken.tRPAREN); + validateToken(IToken.tLBRACE); + validateIdentifier("balance"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateInteger("5000"); //$NON-NLS-1$ + validateToken(IToken.tSEMI); + validateToken(IToken.tRBRACE); + validateToken(IToken.tSEMI); + + validateToken(IToken.t_if); + validateToken(IToken.tLPAREN); + validateIdentifier("confused"); //$NON-NLS-1$ + validateToken(IToken.tLPAREN); + validateToken(IToken.tRPAREN); + validateToken(IToken.tRPAREN); + validateToken(IToken.tLBRACE); + validateIdentifier("you"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateIdentifier("dumb"); //$NON-NLS-1$ + validateToken(IToken.tSEMI); + validateToken(IToken.tRBRACE); + validateToken(IToken.tSEMI); + validateEOF(); + + initializeScanner("#define ON 7\n#if defined(ON)\nint itsOn = ON;\n#endif"); //$NON-NLS-1$ + validateToken(IToken.t_int); + validateBalance(1); + validateIdentifier("itsOn"); //$NON-NLS-1$ + validateToken(IToken.tASSIGN); + validateInteger("7"); //$NON-NLS-1$ + validateToken(IToken.tSEMI); + validateEOF(); + validateBalance(); + + initializeScanner("#if defined( NOTHING ) \nint x = NOTHING;\n#endif"); //$NON-NLS-1$ + validateEOF(); + validateBalance(); + + + + + + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + } + + public void testQuickScan() throws ParserFactoryError + { + try + { + initializeScanner( "#if X + 5 < 7\n int found = 1;\n#endif", ParserMode.QUICK_PARSE ); //$NON-NLS-1$ + validateToken( IToken.t_int ); + validateIdentifier( "found" ); //$NON-NLS-1$ + validateToken( IToken.tASSIGN ); + validateInteger( "1"); //$NON-NLS-1$ + validateToken( IToken.tSEMI ); + validateEOF(); + + } + catch( ScannerException se ) + { + fail( EXCEPTION_THROWN + se.getMessage() ); + } + + try + { + initializeScanner( "#if 0\n int error = 666;\n#endif" ); //$NON-NLS-1$ + validateEOF(); + } + catch( ScannerException se ) + { + fail( EXCEPTION_THROWN + se.getMessage() ); + } + + } + + + public void testOtherPreprocessorCommands() throws ParserFactoryError + { + try + { + initializeScanner("#\n#\t\n#define MAX_SIZE 1024\n#\n# "); //$NON-NLS-1$ + validateEOF(); + validateDefinition("MAX_SIZE", "1024"); //$NON-NLS-1$ //$NON-NLS-2$ + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + + for (int i= 0; i < 4; ++i) + { + switch (i) + { + case 0 : + initializeScanner("# ape"); //$NON-NLS-1$ + break; + case 1 : + initializeScanner("# #"); //$NON-NLS-1$ + break; + case 2 : + initializeScanner("# 32"); //$NON-NLS-1$ + break; + case 3 : + initializeScanner("# defines"); //$NON-NLS-1$ + break; + } + + try + { + validateEOF(); + fail(EXPECTED_FAILURE); + } + catch (ScannerException se) + { + validateBalance(); + } + catch (Exception e) + { + fail(EXCEPTION_THROWN + e.toString()); + } + } + + } + + public void validateAllDefinitions(TableRow row) + { + int winner= row.selectWinner(); + int rowLength= row.getLength(); + for (int i= 0; i <= rowLength; ++i) + { + if (i == winner) + validateDefinition(row.symbolName(i), row.symbolValue(i)); + else + validateAsUndefined(row.symbolName(i)); + } + } + + public void testBug36287() throws Exception + { + initializeScanner( "X::X( const X & rtg_arg ) : U( rtg_arg ) , Z( rtg_arg.Z ) , er( rtg_arg.er ){}" ); //$NON-NLS-1$ + validateIdentifier("X"); //$NON-NLS-1$ + validateToken( IToken.tCOLONCOLON); + validateIdentifier("X"); //$NON-NLS-1$ + validateToken( IToken.tLPAREN ); + validateToken( IToken.t_const ); + validateIdentifier("X"); //$NON-NLS-1$ + validateToken( IToken.tAMPER ); + validateIdentifier( "rtg_arg"); //$NON-NLS-1$ + validateToken( IToken.tRPAREN ); + validateToken( IToken.tCOLON ); + validateIdentifier( "U"); //$NON-NLS-1$ + validateToken( IToken.tLPAREN ); + validateIdentifier( "rtg_arg"); //$NON-NLS-1$ + validateToken( IToken.tRPAREN ); + validateToken( IToken.tCOMMA ); + validateIdentifier( "Z"); //$NON-NLS-1$ + validateToken( IToken.tLPAREN ); + validateIdentifier( "rtg_arg"); //$NON-NLS-1$ + validateToken( IToken.tDOT ); + validateIdentifier( "Z"); //$NON-NLS-1$ + validateToken( IToken.tRPAREN ); + validateToken( IToken.tCOMMA ); + validateIdentifier( "er"); //$NON-NLS-1$ + validateToken( IToken.tLPAREN ); + validateIdentifier( "rtg_arg"); //$NON-NLS-1$ + validateToken( IToken.tDOT ); + validateIdentifier( "er"); //$NON-NLS-1$ + validateToken( IToken.tRPAREN ); + validateToken( IToken.tLBRACE); + validateToken( IToken.tRBRACE); + validateEOF(); + + initializeScanner( "foo.*bar"); //$NON-NLS-1$ + validateIdentifier("foo"); //$NON-NLS-1$ + validateToken( IToken.tDOTSTAR ); + validateIdentifier("bar"); //$NON-NLS-1$ + validateEOF(); + + initializeScanner( "foo...bar"); //$NON-NLS-1$ + validateIdentifier("foo"); //$NON-NLS-1$ + validateToken( IToken.tELLIPSIS ); + validateIdentifier("bar"); //$NON-NLS-1$ + validateEOF(); + } + + public void testBug35892() throws ParserFactoryError + { + try + { + initializeScanner( "'c'" ); //$NON-NLS-1$ + validateChar( 'c' ); + validateEOF(); + } + catch( ScannerException se ) + { + fail( EXCEPTION_THROWN + se.getMessage() ); + } + } + + public void testBug36045() throws Exception + { + StringBuffer buffer = new StringBuffer(); + buffer.append( '"' ); + buffer.append( '\\'); + buffer.append( '"'); + buffer.append( '"'); + + buffer.append( '"'); + buffer.append( '\\'); + buffer.append( '\\'); + buffer.append( '"'); + buffer.append( "\n\n"); //$NON-NLS-1$ + initializeScanner( buffer.toString()); + validateString( "\\\"\\\\"); //$NON-NLS-1$ + } + + public void testConditionalWithBraces() throws ParserFactoryError + { + try + { + for( int i = 0; i < 4; ++i ) + { + initializeScanner( "int foobar(int a) { if(a == 0) {\n#ifdef THIS\n} else {}\n#elif THAT\n} else {}\n#endif\nreturn 0;}" ); //$NON-NLS-1$ + switch( i ) + { + case 0: + scanner.addDefinition( "THIS", "1"); //$NON-NLS-1$ //$NON-NLS-2$ + scanner.addDefinition( "THAT", "1" ); //$NON-NLS-1$ //$NON-NLS-2$ + break; + case 1: + scanner.addDefinition( "THIS", "1"); //$NON-NLS-1$ //$NON-NLS-2$ + scanner.addDefinition( "THAT", "0" ); //$NON-NLS-1$ //$NON-NLS-2$ + break; + case 2: + scanner.addDefinition( "THAT", "1" ); //$NON-NLS-1$ //$NON-NLS-2$ + break; + case 3: + scanner.addDefinition( "THAT", "0" ); //$NON-NLS-1$ //$NON-NLS-2$ + break; + } + + validateToken( IToken.t_int ); + validateIdentifier( "foobar"); //$NON-NLS-1$ + validateToken( IToken.tLPAREN ); + validateToken( IToken.t_int ); + validateIdentifier( "a" ); //$NON-NLS-1$ + validateToken( IToken.tRPAREN ); + validateToken( IToken.tLBRACE ); + validateToken( IToken.t_if ); + validateToken( IToken.tLPAREN ); + validateIdentifier( "a" ); //$NON-NLS-1$ + validateToken( IToken.tEQUAL ); + validateInteger( "0" ); //$NON-NLS-1$ + validateToken( IToken.tRPAREN ); + validateToken( IToken.tLBRACE ); + + if( i <= 1 ) + { + validateToken( IToken.tRBRACE ); + validateToken( IToken.t_else ); + validateToken( IToken.tLBRACE ); + validateToken( IToken.tRBRACE ); + } + + if( i == 2 ) + { + validateToken( IToken.tRBRACE ); + validateToken( IToken.t_else ); + validateToken( IToken.tLBRACE ); + validateToken( IToken.tRBRACE ); + } + + validateToken( IToken.t_return ); + validateInteger( "0"); //$NON-NLS-1$ + validateToken( IToken.tSEMI ); + validateToken( IToken.tRBRACE ); + validateEOF(); + } + } catch( ScannerException se ) + { + fail(EXCEPTION_THROWN + se.toString()); + } + } + + public void testNestedRecursiveDefines() throws Exception + { + initializeScanner( "#define C B A\n#define B C C\n#define A B\nA" ); //$NON-NLS-1$ + + validateIdentifier("B"); //$NON-NLS-1$ + validateDefinition("A", "B"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("B", "C C"); //$NON-NLS-1$ //$NON-NLS-2$ + validateDefinition("C", "B A"); //$NON-NLS-1$ //$NON-NLS-2$ + validateIdentifier("A"); //$NON-NLS-1$ + validateIdentifier("B"); //$NON-NLS-1$ + validateIdentifier("A"); //$NON-NLS-1$ + validateEOF(); + } + + public void testBug36316() throws Exception + { + initializeScanner( "#define A B->A\nA" ); //$NON-NLS-1$ + + validateIdentifier("B"); //$NON-NLS-1$ + validateDefinition("A", "B->A"); //$NON-NLS-1$ //$NON-NLS-2$ + validateToken(IToken.tARROW); + validateIdentifier("A"); //$NON-NLS-1$ + validateEOF(); + } + + public void testBug36434() throws Exception + { + initializeScanner( "#define X(Y)"); //$NON-NLS-1$ + validateEOF(); + IMacroDescriptor macro = scanner.getDefinition( "X" ); //$NON-NLS-1$ + assertNotNull( macro ); + assertEquals( macro.getParameters().length, 1 ); + assertEquals( macro.getParameters()[0], "Y" ); //$NON-NLS-1$ + assertEquals( macro.getTokenizedExpansion().length, 0 ); + } + + public void testBug36047() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write( "# define MAD_VERSION_STRINGIZE(str) #str\n" ); //$NON-NLS-1$ + writer.write( "# define MAD_VERSION_STRING(num) MAD_VERSION_STRINGIZE(num)\n" ); //$NON-NLS-1$ + writer.write( "# define MAD_VERSION MAD_VERSION_STRING(MAD_VERSION_MAJOR) \".\" \\\n" ); //$NON-NLS-1$ + writer.write( " MAD_VERSION_STRING(MAD_VERSION_MINOR) \".\" \\\n" ); //$NON-NLS-1$ + writer.write( " MAD_VERSION_STRING(MAD_VERSION_PATCH) \".\" \\\n" ); //$NON-NLS-1$ + writer.write( " MAD_VERSION_STRING(MAD_VERSION_EXTRA)\n" ); //$NON-NLS-1$ + writer.write( "# define MAD_VERSION_MAJOR 2\n" ); //$NON-NLS-1$ + writer.write( "# define MAD_VERSION_MINOR 1\n" ); //$NON-NLS-1$ + writer.write( "# define MAD_VERSION_PATCH 3\n" ); //$NON-NLS-1$ + writer.write( "# define MAD_VERSION_EXTRA boo\n" ); //$NON-NLS-1$ + writer.write( "MAD_VERSION\n" ); //$NON-NLS-1$ + initializeScanner( writer.toString() ); + + validateString( "2.1.3.boo" ); //$NON-NLS-1$ + + validateEOF(); + } + + public void testBug36475() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write( " \"A\" \"B\" \"C\" " ); //$NON-NLS-1$ + + initializeScanner( writer.toString() ); + + validateString( "ABC" ); //$NON-NLS-1$ + validateEOF(); + } + + public void testBug36509() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write("#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\\n"); //$NON-NLS-1$ + writer.write(" x ## s, x ## t) \n"); //$NON-NLS-1$ + writer.write("debug(1, 2);"); //$NON-NLS-1$ + + initializeScanner( writer.toString() ); + //printf("x1=%d, x2= %s", x1, x2); + validateIdentifier( "printf" ); //$NON-NLS-1$ + validateToken( IToken.tLPAREN ); + validateString("x1= %d, x2= %s"); //$NON-NLS-1$ + validateToken(IToken.tCOMMA); + validateIdentifier("x1"); //$NON-NLS-1$ + validateToken(IToken.tCOMMA); + validateIdentifier("x2"); //$NON-NLS-1$ + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + validateEOF(); + } + + public void testBug36695() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write("\'\\4\' \'\\n\'"); //$NON-NLS-1$ + initializeScanner( writer.toString() ); + + validateChar( "\\4" ); //$NON-NLS-1$ + validateChar( "\\n" ); //$NON-NLS-1$ + validateEOF(); + } + + public void testBug36521() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write("#define str(s) # s\n"); //$NON-NLS-1$ + writer.write("fputs(str(strncmp(\"abc\\0d\", \"abc\", \'\\4\')\n"); //$NON-NLS-1$ + writer.write(" == 0), s);\n"); //$NON-NLS-1$ + + initializeScanner( writer.toString() ); + validateIdentifier("fputs"); //$NON-NLS-1$ + validateToken(IToken.tLPAREN); + validateString("strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0"); //$NON-NLS-1$ + validateToken(IToken.tCOMMA); + validateIdentifier("s"); //$NON-NLS-1$ + validateToken(IToken.tRPAREN); + validateToken(IToken.tSEMI); + } + + public void testBug36770() throws Exception + { + StringWriter writer = new StringWriter(); + writer.write( "#define A 0\n" ); //$NON-NLS-1$ + writer.write( "#if ( A == 1 )\n"); //$NON-NLS-1$ + writer.write( "# define foo 1\n"); //$NON-NLS-1$ + writer.write( "#else\n"); //$NON-NLS-1$ + writer.write( "# define foo 2\n"); //$NON-NLS-1$ + writer.write( "#endif\n"); //$NON-NLS-1$ + writer.write( "foo\n"); //$NON-NLS-1$ + initializeScanner( writer.toString() ); + validateInteger( "2" ); //$NON-NLS-1$ + validateEOF(); + } + + public void testBug36816() throws Exception + { + initializeScanner( "#include \"foo.h" ); //$NON-NLS-1$ + try{ + validateEOF(); + } catch ( ScannerException e ){ + assertTrue( e.getProblem().getID() == IProblem.PREPROCESSOR_INVALID_DIRECTIVE ); + } + + initializeScanner( "#include \n" ); //$NON-NLS-1$ + + Callback callback = new Callback( ParserMode.QUICK_PARSE ); + initializeScanner( buffer.toString(), ParserMode.QUICK_PARSE, callback ); + validateEOF(); + assertEquals( callback.problems.size(), 0 ); + assertEquals( callback.inclusions.size(), 1 ); + assertEquals( callback.inclusions.get(0), "stdio.h"); //$NON-NLS-1$ + } + + public void testBug46402() throws Exception + { + StringBuffer buffer = new StringBuffer(); + buffer.append( "#define X 5\n" ); //$NON-NLS-1$ + buffer.append( "#if defined( X )\n" ); //$NON-NLS-1$ + buffer.append( "// blah\n" ); //$NON-NLS-1$ + buffer.append( "#elif Y > 5 \n" ); //$NON-NLS-1$ + buffer.append( "// coo\n" ); //$NON-NLS-1$ + buffer.append( "#endif\n" ); //$NON-NLS-1$ + initializeScanner( buffer.toString(), ParserMode.COMPLETE_PARSE ); + validateEOF(); + } + + public void testBug50821() throws Exception + { + Callback callback = new Callback( ParserMode.QUICK_PARSE ); + initializeScanner( "\'\n\n\n", ParserMode.QUICK_PARSE, callback ); //$NON-NLS-1$ + scanner.nextToken(); + assertEquals( callback.problems.size(), 1 ); + } + + + public void test54778() throws ScannerException + { + initializeScanner("#if 1 || 0 < 3 \n printf \n #endif\n"); //$NON-NLS-1$ + validateIdentifier("printf"); //$NON-NLS-1$ + validateEOF(); + initializeScanner("#if !defined FOO || FOO > 3\nprintf\n#endif\n"); //$NON-NLS-1$ + validateIdentifier("printf"); //$NON-NLS-1$ + validateEOF(); + initializeScanner("#if !defined FOO || FOO < 3\nprintf\n#endif\n"); //$NON-NLS-1$ + validateIdentifier("printf"); //$NON-NLS-1$ + validateEOF(); + + } + + public void testBug56517() throws Exception + { + Writer writer = new StringWriter(); + writer.write( "#if 0 \n"); //$NON-NLS-1$ + writer.write( "char * x = \"#boo\";\n" ); //$NON-NLS-1$ + writer.write( "#endif\n"); //$NON-NLS-1$ + initializeScanner( writer.toString() ); + validateEOF(); + } + + public void testBug36770B() throws Exception + { + Writer writer = new StringWriter(); + writer.write( "#define A 0\n" ); //$NON-NLS-1$ + writer.write( "#if ( A == 1 )\n" ); //$NON-NLS-1$ + writer.write( "# define foo\n" ); //$NON-NLS-1$ + writer.write( "#else\n" ); //$NON-NLS-1$ + writer.write( "# define bar\n" ); //$NON-NLS-1$ + writer.write( "#endif\n" ); //$NON-NLS-1$ + initializeScanner( writer.toString(), ParserMode.QUICK_PARSE ); + validateEOF(); + validateDefinition( "A", 0 ); //$NON-NLS-1$ + validateDefinition( "bar", "" ); //$NON-NLS-1$ //$NON-NLS-2$ + + } + + public void testBug47797() throws Exception + { + initializeScanner( "\"\\uABCD\" \'\\uABCD\' \\uABCD_ident \\u001A01BC_ident ident\\U01AF ident\\u01bc00AF"); //$NON-NLS-1$ + validateString( "\\uABCD"); //$NON-NLS-1$ + validateChar( "\\uABCD"); //$NON-NLS-1$ + validateIdentifier( "\\uABCD_ident"); //$NON-NLS-1$ + validateIdentifier( "\\u001A01BC_ident"); //$NON-NLS-1$ + validateIdentifier( "ident\\U01AF" ); //$NON-NLS-1$ + validateIdentifier( "ident\\u01bc00AF" ); //$NON-NLS-1$ + validateEOF(); + } + + public void testBug39698() throws Exception + { + initializeScanner( "?"); //$NON-NLS-1$ + validateToken( IGCCToken.tMIN ); + validateToken( IGCCToken.tMAX ); + validateEOF(); + } + + public void testBug59768() throws Exception + { + initializeScanner( "#define A A\nA"); //$NON-NLS-1$ + validateIdentifier( "A"); //$NON-NLS-1$ + validateEOF(); + IMacroDescriptor d = scanner.getDefinition( "A"); //$NON-NLS-1$ + assertTrue( d.isCircular() ); + } + + public void testBug60764() throws Exception + { + Writer writer = new StringWriter(); + writer.write( "#define P a,b\n"); //$NON-NLS-1$ + writer.write( "#define M(x) M1(x)\n"); //$NON-NLS-1$ + writer.write( "#define M1(x,y) #x #y\n"); //$NON-NLS-1$ + writer.write( "M(P)\n"); //$NON-NLS-1$ + initializeScanner( writer.toString() ); + validateString( "ab"); //$NON-NLS-1$ + validateEOF(); + } + + public void testBug62042() throws Exception + { + Callback callback = new Callback(ParserMode.QUICK_PARSE); + initializeScanner( "0x", ParserMode.QUICK_PARSE, callback ); //$NON-NLS-1$ + validateEOF(); + assertFalse( callback.problems.isEmpty() ); + } + + public void testBug61968() throws Exception + { + Writer writer = new StringWriter(); + writer.write( "unsigned int ui = 2172748163; //ok \n" ); //$NON-NLS-1$ + writer.write( "int big = 999999999999999;//ok \n" ); //$NON-NLS-1$ + writer.write( "void main() { \n" ); //$NON-NLS-1$ + writer.write( "caller(4); //ok\n" ); //$NON-NLS-1$ + writer.write( "caller(2172748163);//causes java.lang.NumberFormatException \n" ); //$NON-NLS-1$ + writer.write( "caller(999999999999999); //also causes NumberFormatException \n" ); //$NON-NLS-1$ + writer.write( "}\n" ); //$NON-NLS-1$ + Callback callback = new Callback(ParserMode.QUICK_PARSE); + initializeScanner( writer.toString(), ParserMode.QUICK_PARSE, callback ); + fullyTokenize(); + assertTrue( callback.problems.isEmpty() ); + } + + public void testBug62378() throws Exception + { + initializeScanner( "\"\\?\\?<\""); //$NON-NLS-1$ + validateString("\\?\\?<" ); //$NON-NLS-1$ + } + + public void testBug62384() throws Exception + { + initializeScanner( "18446744073709551615LL"); //$NON-NLS-1$ + validateInteger( "18446744073709551615"); //$NON-NLS-1$ + } + + public void testBug62390() throws Exception + { + Writer writer = new StringWriter(); + writer.write( "#define f(x) x\n"); //$NON-NLS-1$ + writer.write( "#if f(\n"); //$NON-NLS-1$ + writer.write( "5) == 5\n"); //$NON-NLS-1$ + writer.write( "true1\n"); //$NON-NLS-1$ + writer.write( "#endif\n"); //$NON-NLS-1$ + writer.write( "#if A\n"); //$NON-NLS-1$ + writer.write( "#elif f(\n"); //$NON-NLS-1$ + writer.write( "5) == 5\n"); //$NON-NLS-1$ + writer.write( "true2\n"); //$NON-NLS-1$ + writer.write( "#endif\n"); //$NON-NLS-1$ + writer.write( "#undef f\n"); //$NON-NLS-1$ + writer.write( "#define f(x) \"A0I70_001.h\"\n"); //$NON-NLS-1$ + writer.write( "#include f(\n"); //$NON-NLS-1$ + writer.write( "5\n"); //$NON-NLS-1$ + writer.write( ")\n"); //$NON-NLS-1$ + writer.write( "#undef f\n"); //$NON-NLS-1$ + writer.write( "#define f(x) 1467\n"); //$NON-NLS-1$ + writer.write( "#line f(\n"); //$NON-NLS-1$ + writer.write( "5\n"); //$NON-NLS-1$ + writer.write( ")\n"); //$NON-NLS-1$ + writer.write( "#pragma f(\n"); //$NON-NLS-1$ + writer.write( "5\n"); //$NON-NLS-1$ + writer.write( ")\n"); //$NON-NLS-1$ + writer.write( "}\n"); //$NON-NLS-1$ + Callback callback = new Callback( ParserMode.QUICK_PARSE ); + initializeScanner( writer.toString(), ParserMode.QUICK_PARSE, callback ); + fullyTokenize(); + } + + public void testBug62009() throws Exception + { + Callback callback = new Callback( ParserMode.QUICK_PARSE ); + initializeScanner( "#define def(x) (x#)\ndef(orange)\n", ParserMode.QUICK_PARSE, callback ); //$NON-NLS-1$ + fullyTokenize(); + assertFalse( callback.problems.isEmpty() ); + } + + public void testBug61972() throws Exception + { + initializeScanner( "#define DEF1(A1) A1\n#define DEF2 DEF1(DEF2)\nDEF2;" ); //$NON-NLS-1$ + validateIdentifier( "DEF2"); //$NON-NLS-1$ + validateToken( IToken.tSEMI ); + validateEOF(); + } + + public void testBug64268() throws Exception + { + Writer writer = new StringWriter(); + writer.write("#define BODY \\\n"); //$NON-NLS-1$ + writer.write(" { \\\n"); //$NON-NLS-1$ + writer.write(" /* this multi-line comment messes \\\n"); //$NON-NLS-1$ + writer.write(" up the parser. */ }\n"); //$NON-NLS-1$ + writer.write("BODY "); //$NON-NLS-1$ + initializeScanner( writer.toString() ); + validateToken( IToken.tLBRACE); + validateToken( IToken.tRBRACE); + validateEOF(); + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CharArrayMap.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CharArrayMap.java deleted file mode 100644 index 46d699c185b..00000000000 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CharArrayMap.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Created on May 28, 2004 - * - * TODO To change the template for this generated file go to - * Window - Preferences - Java - Code Style - Code Templates - */ -package org.eclipse.cdt.core.parser; - -/** - * @author dschaefe - * - * TODO To change the template for this generated type comment go to - * Window - Preferences - Java - Code Style - Code Templates - */ -public class CharArrayMap { - - private char[][] keyTable; - private Object[] valueTable; - private int[] hashTable; - private int[] nextTable; - - private int currEntry; - - public CharArrayMap(int initialSize) { - // Make sure size is a power of two - int size = 1; - while (size < initialSize) - size <<= 1; - createTables(size); - } - - private final void createTables(int size) { - keyTable = new char[size][]; - valueTable = new Object[size]; - nextTable = new int[size]; - hashTable = new int[size << 1]; - currEntry = 0; - } - - private final int hash(char[] key) { - return CharArrayUtils.hash(key) & (hashTable.length - 1); - } - - private final int add(char[] key, Object value) { - keyTable[currEntry] = key; - valueTable[currEntry] = value; - return ++currEntry; - } - - // returns the overwritten object if there was one - public Object put(char[] key, Object value) { - try { - int hash = hash(key); - int i = hashTable[hash] - 1; - if (i < 0) { - // Nobody here - hashTable[hash] = add(key, value); - } else { - // See if the key is already defined - int last = i; - while (i >= 0) { - if (CharArrayUtils.equals(key, keyTable[i])) { - Object oldvalue = valueTable[i]; - valueTable[i] = value; - // Nothing left to do, escape... - return oldvalue; - } - last = i; - i = nextTable[i] - 1; - } - - // Not there, time to add - nextTable[last] = add(key, value); - } - - return null; - } catch (IndexOutOfBoundsException e) { - // Oops, too many, resize and try again - char[][] oldKeyTable = keyTable; - Object[] oldValueTable = valueTable; - - int newSize = hashTable.length << 1; - createTables(newSize); - for (int i = 0; i < oldKeyTable.length; ++i) - put(oldKeyTable[i], oldValueTable[i]); - - return put(key, value); - } - } - - public Object get(char[] key) { - int hash = hash(key); - int i = hashTable[hash] - 1; - while (i >= 0) { - if (CharArrayUtils.equals(key, keyTable[i])) - return valueTable[i]; - i = nextTable[i] - 1; - } - return null; - } -} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java index ffedab54f45..8a8703c5274 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java @@ -309,6 +309,9 @@ public interface IToken { static public final int t_restrict = 137; - static public final int tLAST = t_restrict; + static public final int tMACROEXP = 138; + + static public final int tPOUNDPOUND = 139; + static public final int tLAST = 139; } \ No newline at end of file diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java index bdf1a71590e..ecb4af31318 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerUtility.java @@ -21,7 +21,7 @@ import org.eclipse.cdt.core.parser.ISourceElementRequestor; */ public class ScannerUtility { - static String reconcilePath(String originalPath ) { + public static String reconcilePath(String originalPath ) { if( originalPath == null ) return null; originalPath = removeQuotes( originalPath ); @@ -66,7 +66,7 @@ public class ScannerUtility { } - static CodeReader createReaderDuple( String path, ISourceElementRequestor requestor, Iterator workingCopies ) + public static CodeReader createReaderDuple( String path, ISourceElementRequestor requestor, Iterator workingCopies ) { return requestor.createReader( path, workingCopies ); } @@ -85,7 +85,7 @@ public class ScannerUtility { return reconcilePath( newPathBuffer.toString() ); } - static class InclusionDirective + public static class InclusionDirective { public InclusionDirective( String fileName, boolean useIncludePaths, int startOffset, int endOffset ) { @@ -121,7 +121,7 @@ public class ScannerUtility { } } - static class InclusionParseException extends Exception + public static class InclusionParseException extends Exception { } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayIntMap.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayIntMap.java new file mode 100644 index 00000000000..f97d6ea459c --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayIntMap.java @@ -0,0 +1,50 @@ +/********************************************************************** + * Copyright (c) 2004 IBM and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v10.html + * + * Contributors: + * IBM - Initial implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner2; + +/** + * @author Doug Schaefer + */ +public class CharArrayIntMap extends CharArrayMap { + + private int[] valueTable; + + public CharArrayIntMap(int initialSize) { + super(initialSize); + valueTable = new int[capacity()]; + } + + protected void resize(int size) { + int[] oldValueTable = valueTable; + valueTable = new int[size]; + System.arraycopy(oldValueTable, 0, valueTable, 0, oldValueTable.length); + } + + public int put(char[] key, int start, int length, int value) { + int i = add(key, start, length); + int oldvalue = valueTable[i]; + valueTable[i] = value; + return oldvalue; + } + + public int put(char[] key, int value) { + return put(key, 0, key.length, value); + } + + public int get(char[] key, int start, int length) { + int i = lookup(key, start, length); + if (i >= 0) + return valueTable[i]; + else + return -1; + } + +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayMap.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayMap.java new file mode 100644 index 00000000000..e4317238181 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayMap.java @@ -0,0 +1,137 @@ +/********************************************************************** + * Copyright (c) 2004 IBM and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v10.html + * + * Contributors: + * IBM - Initial implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner2; + +/** + * @author Doug Schaefer + */ +public abstract class CharArrayMap { + + private char[][] keyTable; + private int[] hashTable; + private int[] nextTable; + protected int currEntry = -1; + + protected CharArrayMap(int initialSize) { + int size = 1; + while (size < initialSize) + size <<= 1; + + keyTable = new char[size][]; + hashTable = new int[size * 2]; + nextTable = new int[size]; + } + + protected int capacity() { + return keyTable.length; + } + + private int hash(char[] buffer, int start, int len) { + return CharArrayUtils.hash(buffer, start, len) & (keyTable.length - 1); + } + + private void insert(int i) { + insert(i, hash(keyTable[i], 0, keyTable[i].length)); + } + + private void insert(int i, int hash) { + + if (hashTable[hash] == 0) { + hashTable[hash] = i + 1; + } else { + // need to link + int j = hashTable[hash] - 1; + while (nextTable[j] != 0) + j = nextTable[j] - 1; + nextTable[j] = i + 1; + } + } + + protected void resize(int size) { + char[][] oldKeyTable = keyTable; + keyTable = new char[size][]; + System.arraycopy(oldKeyTable, 0, keyTable, 0, oldKeyTable.length); + + // Need to rehash everything + hashTable = new int[size * 2]; + nextTable = new int[size]; + for (int i = 0; i < oldKeyTable.length; ++i) { + insert(i); + } + } + + private void resize() { + resize(keyTable.length << 1); + } + + protected final int add(char[] buffer, int start, int len) { + int hash = hash(buffer, start, len); + + if (hashTable[hash] == 0) { + keyTable[++currEntry] = CharArrayUtils.extract(buffer, start, len); + insert(currEntry, hash); + return currEntry; + } else { + // is the key already registered? + int i = hashTable[hash] - 1; + if (CharArrayUtils.equals(buffer, start, len, keyTable[i])) + // yup + return i; + + // follow the next chain + int last = i; + for (i = nextTable[i] - 1; i >= 0; i = nextTable[i] - 1) { + if (CharArrayUtils.equals(buffer, start, len, keyTable[i])) + // yup this time + return i; + last = i; + } + + // nope, add it in + keyTable[++currEntry] = CharArrayUtils.extract(buffer, start, len); + nextTable[last] = currEntry + 1; + return currEntry; + } + } + + protected final int lookup(char[] buffer, int start, int len) { + int hash = hash(buffer, start, len); + + if (hashTable[hash] == 0) + return -1; + + int i = hashTable[hash] - 1; + if (CharArrayUtils.equals(buffer, start, len, keyTable[i])) + return i; + + // Follow the next chain + for (i = nextTable[i] - 1; i >= 0; i = nextTable[i] - 1) + if (CharArrayUtils.equals(buffer, start, len, keyTable[i])) + return i; + + return -1; + } + + public void dumpNexts() { + for (int i = 0; i < nextTable.length; ++i) { + if (nextTable[i] == 0) + continue; + + System.out.print(i); + + for (int j = nextTable[i] - 1; j >= 0; j = nextTable[j] - 1) + System.out.print(" -> " + j); + + System.out.println(""); + } + } + +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayObjectMap.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayObjectMap.java new file mode 100644 index 00000000000..a501afedbcd --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayObjectMap.java @@ -0,0 +1,54 @@ +/********************************************************************** + * Copyright (c) 2004 IBM and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v10.html + * + * Contributors: + * IBM - Initial implementation +***********************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner2; + +/** + * @author Doug Schaefer + */ +public class CharArrayObjectMap extends CharArrayMap { + + private Object[] valueTable; + + public CharArrayObjectMap(int initialSize) { + super(initialSize); + valueTable = new Object[capacity()]; + } + + protected void resize(int size) { + Object[] oldValueTable = valueTable; + valueTable = new Object[size]; + System.arraycopy(oldValueTable, 0, valueTable, 0, oldValueTable.length); + } + + public Object put(char[] key, int start, int length, Object value) { + int i = add(key, start, length); + Object oldvalue = valueTable[i]; + valueTable[i] = value; + return oldvalue; + } + + public Object put(char[] key, Object value) { + return put(key, 0, key.length, value); + } + + public Object get(char[] key, int start, int length) { + int i = lookup(key, start, length); + if (i >= 0) + return valueTable[i]; + else + return null; + } + + public Object get(char[] key) { + return get(key, 0, key.length); + } + +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CharArrayPool.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayPool.java similarity index 98% rename from core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CharArrayPool.java rename to core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayPool.java index 0a0c6453c78..0af7fb2c7b3 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CharArrayPool.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayPool.java @@ -4,7 +4,7 @@ * TODO To change the template for this generated file go to * Window - Preferences - Java - Code Style - Code Templates */ -package org.eclipse.cdt.core.parser; +package org.eclipse.cdt.internal.core.parser.scanner2; /** * @author dschaefe diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CharArrayUtils.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayUtils.java similarity index 63% rename from core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CharArrayUtils.java rename to core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayUtils.java index 028b90abb36..a7dca813949 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CharArrayUtils.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/CharArrayUtils.java @@ -4,7 +4,7 @@ * TODO To change the template for this generated file go to * Window - Preferences - Java - Code Style - Code Templates */ -package org.eclipse.cdt.core.parser; +package org.eclipse.cdt.internal.core.parser.scanner2; /** * @author dschaefe @@ -32,9 +32,6 @@ public class CharArrayUtils { if (str1 == str2) return true; - if (str1 == null || str2 == null) - return false; - if (str1.length != str2.length) return false; @@ -44,4 +41,24 @@ public class CharArrayUtils { return true; } + + public static final boolean equals(char[] str1, int start1, int length1, char[] str2) { + if (length1 != str2.length) + return false; + + for (int i = 0; i < length1; ++i) + if (str1[start1++] != str2[i]) + return false; + + return true; + } + + public static final char[] extract(char[] str, int start, int length) { + if (start == 0 && length == str.length) + return str; + + char[] copy = new char[length]; + System.arraycopy(str, start, copy, 0, length); + return copy; + } } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/FunctionStyleMacro.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/FunctionStyleMacro.java new file mode 100644 index 00000000000..ad05cddab9c --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/FunctionStyleMacro.java @@ -0,0 +1,31 @@ +/******************************************************************************* + * Copyright (c) 2004 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Corporation - initial implementation + ******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner2; + +/** + * @author Doug Schaefer + */ +public class FunctionStyleMacro extends ObjectStyleMacro { + + public char[][] arglist; + + public FunctionStyleMacro(char[] name, char[] expansion, char[][] arglist) { + super(name, expansion); + this.arglist = arglist; + } + + public class Expansion { + + public final CharArrayObjectMap definitions + = new CharArrayObjectMap(FunctionStyleMacro.this.arglist.length); + + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/MacroExpansionToken.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/MacroExpansionToken.java new file mode 100644 index 00000000000..0c2e27dd903 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/MacroExpansionToken.java @@ -0,0 +1,134 @@ +/******************************************************************************* + * Copyright (c) 2004 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Corporation - initial implementation + ******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner2; + +import org.eclipse.cdt.core.parser.IToken; + +/** + * @author Doug Schaefer + */ +public class MacroExpansionToken implements IToken { + + public MacroExpansionToken() { + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#getType() + */ + public int getType() { + return IToken.tMACROEXP; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#getImage() + */ + public String getImage() { + // TODO Auto-generated method stub + return null; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#getOffset() + */ + public int getOffset() { + // TODO Auto-generated method stub + return 0; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#getLength() + */ + public int getLength() { + // TODO Auto-generated method stub + return 0; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#getEndOffset() + */ + public int getEndOffset() { + // TODO Auto-generated method stub + return 0; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#getLineNumber() + */ + public int getLineNumber() { + // TODO Auto-generated method stub + return 0; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#getNext() + */ + public IToken getNext() { + // TODO Auto-generated method stub + return null; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#setImage(java.lang.String) + */ + public void setImage(String i) { + // TODO Auto-generated method stub + + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#setNext(org.eclipse.cdt.core.parser.IToken) + */ + public void setNext(IToken t) { + // TODO Auto-generated method stub + + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#setType(int) + */ + public void setType(int i) { + // TODO Auto-generated method stub + + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#looksLikeExpression() + */ + public boolean looksLikeExpression() { + // TODO Auto-generated method stub + return false; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#isPointer() + */ + public boolean isPointer() { + // TODO Auto-generated method stub + return false; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#isOperator() + */ + public boolean isOperator() { + // TODO Auto-generated method stub + return false; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IToken#canBeAPrefix() + */ + public boolean canBeAPrefix() { + // TODO Auto-generated method stub + return false; + } + +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/ObjectStyleMacro.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/ObjectStyleMacro.java new file mode 100644 index 00000000000..170ad373239 --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/ObjectStyleMacro.java @@ -0,0 +1,25 @@ +/******************************************************************************* + * Copyright (c) 2004 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Corporation - initial implementation + ******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner2; + +/** + * @author Doug Schaefer + */ +public class ObjectStyleMacro { + + public char[] name; + public char[] expansion; + + public ObjectStyleMacro(char[] name, char[] expansion) { + this.name = name; + this.expansion = expansion; + } +} diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java new file mode 100644 index 00000000000..dfb6334a38c --- /dev/null +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/Scanner2.java @@ -0,0 +1,1535 @@ +/******************************************************************************* + * Copyright (c) 2004 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Common Public License v0.5 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/cpl-v05.html + * + * Contributors: + * IBM Corporation - initial implementation + ******************************************************************************/ +package org.eclipse.cdt.internal.core.parser.scanner2; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.eclipse.cdt.core.parser.CodeReader; +import org.eclipse.cdt.core.parser.EndOfFileException; +import org.eclipse.cdt.core.parser.IMacroDescriptor; +import org.eclipse.cdt.core.parser.IParserLogService; +import org.eclipse.cdt.core.parser.IScanner; +import org.eclipse.cdt.core.parser.IScannerInfo; +import org.eclipse.cdt.core.parser.ISourceElementRequestor; +import org.eclipse.cdt.core.parser.IToken; +import org.eclipse.cdt.core.parser.ParserLanguage; +import org.eclipse.cdt.core.parser.ParserMode; +import org.eclipse.cdt.core.parser.ScannerException; +import org.eclipse.cdt.core.parser.ast.IASTFactory; +import org.eclipse.cdt.core.parser.extension.IScannerExtension; +import org.eclipse.cdt.internal.core.parser.problem.IProblemFactory; +import org.eclipse.cdt.internal.core.parser.scanner.BranchTracker; +import org.eclipse.cdt.internal.core.parser.scanner.ContextStack; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext; +import org.eclipse.cdt.internal.core.parser.scanner.IScannerData; +import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionDirective; +import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionParseException; +import org.eclipse.cdt.internal.core.parser.token.ImagedToken; +import org.eclipse.cdt.internal.core.parser.token.SimpleToken; + +/** + * @author Doug Schaefer + * + */ +public class Scanner2 implements IScanner, IScannerData { + + private ISourceElementRequestor requestor; + private ParserMode parserMode; + private ParserLanguage language; + protected IParserLogService log; + private IScannerExtension scannerExtension; + private List workingCopies; + private CharArrayObjectMap definitions; + private String[] includePaths; + int count; + + // The context stack + private static final int bufferInitialSize = 8; + private int bufferStackPos = -1; + private char[][] bufferStack = new char[bufferInitialSize][]; + private Object[] bufferData = new Object[bufferInitialSize]; + private int[] bufferPos = new int[bufferInitialSize]; + private int[] bufferLimit = new int[bufferInitialSize]; + + // Utility + private static String[] emptyStringArray = new String[0]; + private static char[] emptyCharArray = new char[0]; + private static EndOfFileException EOF = new EndOfFileException(); + + public Scanner2(CodeReader reader, + IScannerInfo info, + ISourceElementRequestor requestor, + ParserMode parserMode, + ParserLanguage language, + IParserLogService log, + IScannerExtension extension, + List workingCopies) { + + this.scannerExtension = extension; + this.requestor = requestor; + this.parserMode = parserMode; + this.language = language; + this.log = log; + this.workingCopies = workingCopies; + + pushContext(reader.buffer); + + if (info.getDefinedSymbols() != null) { + Map symbols = info.getDefinedSymbols(); + String[] keys = (String[])symbols.keySet().toArray(emptyStringArray); + definitions = new CharArrayObjectMap(symbols.size()); + for (int i = 0; i < keys.length; ++i) { + String symbolName = (String)keys[i]; + Object value = symbols.get(symbolName); + + if( value instanceof String ) { + //TODO add in check here for '(' and ')' + addDefinition( symbolName, scannerExtension.initializeMacroValue(this, (String) value)); + } else if( value instanceof IMacroDescriptor ) + addDefinition( symbolName, (IMacroDescriptor)value); + } + } else { + definitions = new CharArrayObjectMap(4); + } + + includePaths = info.getIncludePaths(); + + } + + private void pushContext(char[] buffer) { + if (++bufferStackPos == bufferStack.length) { + int size = bufferStack.length * 2; + + char[][] oldBufferStack = bufferStack; + bufferStack = new char[size][]; + System.arraycopy(oldBufferStack, 0, bufferStack, 0, oldBufferStack.length); + + Object[] oldBufferData = bufferData; + bufferData = new Object[size]; + System.arraycopy(oldBufferData, 0, bufferData, 0, oldBufferData.length); + + int[] oldBufferPos = bufferPos; + bufferPos = new int[size]; + System.arraycopy(oldBufferPos, 0, bufferPos, 0, oldBufferPos.length); + + int[] oldBufferLimit = bufferLimit; + bufferLimit = new int[size]; + System.arraycopy(oldBufferLimit, 0, bufferLimit, 0, oldBufferLimit.length); + } + + bufferStack[bufferStackPos] = buffer; + bufferPos[bufferStackPos] = -1; + bufferLimit[bufferStackPos] = buffer.length; + } + + private void pushContext(char[] buffer, Object data) { + pushContext(buffer); + bufferData[bufferStackPos] = data; + } + + private void popContext() { + bufferStack[bufferStackPos] = null; + --bufferStackPos; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#addDefinition(java.lang.String, org.eclipse.cdt.core.parser.IMacroDescriptor) + */ + public void addDefinition(String key, IMacroDescriptor macroToBeAdded) { + //definitions.put(key.toCharArray(), macroToBeAdded); + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#addDefinition(java.lang.String, java.lang.String) + */ + public void addDefinition(String key, String value) { + definitions.put(key.toCharArray(), value.toCharArray()); + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#getCount() + */ + public int getCount() { + return count; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#getDefinition(java.lang.String) + */ + public IMacroDescriptor getDefinition(String key) { + return (IMacroDescriptor)definitions.get(key.toCharArray()); + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#getDefinitions() + */ + public Map getDefinitions() { + return null; + } + + public CharArrayObjectMap getRealDefinitions() { + return definitions; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#getDepth() + */ + public int getDepth() { + // TODO Auto-generated method stub + return 0; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#getIncludePaths() + */ + public String[] getIncludePaths() { + return includePaths; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#isOnTopContext() + */ + public boolean isOnTopContext() { + // TODO Auto-generated method stub + return false; + } + + private IToken nextToken; + private boolean finished = false; + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#nextToken() + */ + public IToken nextToken() throws ScannerException, EndOfFileException { + if (finished) + throw EOF; + + if (nextToken == null) { + nextToken = fetchToken(); + if (nextToken == null) + throw EOF; + } + + IToken token = nextToken; + + nextToken = fetchToken(); + if (nextToken == null) + finished = true; + else if (nextToken.getType() == IToken.tPOUNDPOUND) { + // time for a pasting + IToken token2 = fetchToken(); + if (token2 == null) { + nextToken = null; + finished = true; + } else { + String t1 = token.getImage(); + String t2 = token2.getImage(); + char[] pb = new char[t1.length() + t2.length()]; + t1.getChars(0, t1.length(), pb, 0); + t2.getChars(0, t2.length(), pb, t1.length()); + pushContext(pb); + nextToken = null; + return nextToken(); + } + } + + // TODO Check if token is ## and proceed with pasting + + return token; + } + + // Return null to signify end of file + private IToken fetchToken() { + + contextLoop: + while (bufferStackPos >= 0) { + + // Find the first thing we would care about + skipOverWhiteSpace(); + + while (++bufferPos[bufferStackPos] >= bufferLimit[bufferStackPos]) { + // We're at the end of a context, pop it off and try again + popContext(); + continue contextLoop; + } + + // Tokens don't span buffers, stick to our current one + char[] buffer = bufferStack[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + int pos = bufferPos[bufferStackPos]; + + switch (buffer[pos]) { + case 'L': + if (pos + 1 < limit && buffer[pos + 1] == '"') + return scanString(); + else { + IToken t = scanIdentifier(); + if (t instanceof MacroExpansionToken) + continue; + else + return t; + } + + case '"': + return scanString(); + + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + IToken t = scanIdentifier(); + if (t instanceof MacroExpansionToken) + continue; + else + return t; + + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return scanNumber(); + + case '.': + if (pos + 1 < limit) { + switch (buffer[pos + 1]) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return scanNumber(); + + case '.': + if (pos + 2 < limit) { + if (buffer[pos + 2] == '.') { + bufferPos[bufferStackPos] += 2; + return new SimpleToken(IToken.tELLIPSIS); + } + } + case '*': + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tDOTSTAR); + } + } + return new SimpleToken(IToken.tDOT); + + case '#': + if (pos + 1 < limit && buffer[pos + 1] == '#') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tPOUNDPOUND); + } + + // Should really check to make sure this is the first + // non whitespace character on the line + handlePPDirective(); + continue; + + case '{': + return new SimpleToken(IToken.tLBRACE); + + case '}': + return new SimpleToken(IToken.tRBRACE); + + case '[': + return new SimpleToken(IToken.tLBRACKET); + + case ']': + return new SimpleToken(IToken.tRBRACKET); + + case '(': + return new SimpleToken(IToken.tLPAREN); + + case ')': + return new SimpleToken(IToken.tRPAREN); + + case ';': + return new SimpleToken(IToken.tSEMI); + + case ':': + if (pos + 1 < limit) { + if (buffer[pos + 1] == ':') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tCOLONCOLON); + } + } + return new SimpleToken(IToken.tCOLON); + + case '?': + return new SimpleToken(IToken.tQUESTION); + + case '+': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '+') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tINCR); + } else if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tPLUSASSIGN); + } + } + return new SimpleToken(IToken.tPLUS); + + case '-': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '>') { + if (pos + 2 < limit) { + if (buffer[pos + 2] == '*') { + bufferPos[bufferStackPos] += 2; + return new SimpleToken(IToken.tARROWSTAR); + } + } + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tARROW); + } else if (buffer[pos + 1] == '-') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tDECR); + } else if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tMINUSASSIGN); + } + } + return new SimpleToken(IToken.tMINUS); + + case '*': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tSTARASSIGN); + } + } + return new SimpleToken(IToken.tSTAR); + + case '/': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tDIVASSIGN); + } + } + return new SimpleToken(IToken.tDIV); + + case '%': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tMODASSIGN); + } + } + return new SimpleToken(IToken.tMOD); + + case '^': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tXORASSIGN); + } + } + return new SimpleToken(IToken.tXOR); + + case '&': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '&') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tAND); + } else if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tAMPERASSIGN); + } + } + return new SimpleToken(IToken.tAMPER); + + case '|': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '|') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tOR); + } else if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tBITORASSIGN); + } + } + return new SimpleToken(IToken.tBITOR); + + case '~': + return new SimpleToken(IToken.tCOMPL); + + case '!': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tNOTEQUAL); + } + } + return new SimpleToken(IToken.tNOT); + + case '=': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tEQUAL); + } + } + return new SimpleToken(IToken.tASSIGN); + + case '<': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tLTEQUAL); + } else if (buffer[pos + 1] == '<') { + if (pos + 2 < limit) { + if (buffer[pos + 2] == '=') { + bufferPos[bufferStackPos] += 2; + return new SimpleToken(IToken.tSHIFTLASSIGN); + } + } + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tSHIFTL); + } + } + return new SimpleToken(IToken.tLT); + + case '>': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '=') { + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tGTEQUAL); + } else if (buffer[pos + 1] == '>') { + if (pos + 2 < limit) { + if (buffer[pos + 2] == '=') { + bufferPos[bufferStackPos] += 2; + return new SimpleToken(IToken.tSHIFTRASSIGN); + } + } + ++bufferPos[bufferStackPos]; + return new SimpleToken(IToken.tSHIFTR); + } + } + return new SimpleToken(IToken.tGT); + + case ',': + return new SimpleToken(IToken.tCOMMA); + + default: + // skip over anything we don't handle + } + } + + // We've run out of contexts, our work is done here + return null; + } + + private IToken scanIdentifier() { + char[] buffer = bufferStack[bufferStackPos]; + int start = bufferPos[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + int len = 1; + + while (++bufferPos[bufferStackPos] < limit) { + char c = buffer[bufferPos[bufferStackPos]]; + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || c == '_' || (c >= '0' && c <= '9')) { + ++len; + continue; + } else { + break; + } + } + + --bufferPos[bufferStackPos]; + + // Check for macro expansion + Object expObject = null; + if (bufferData[bufferStackPos] instanceof FunctionStyleMacro.Expansion) { + // first check if name is a macro arg + expObject = ((FunctionStyleMacro.Expansion)bufferData[bufferStackPos]) + .definitions.get(buffer, start, len); + } + + if (expObject == null) + // now check regular macros + expObject = definitions.get(buffer, start, len); + + if (expObject != null) { + if (expObject instanceof FunctionStyleMacro) { + handleFunctionStyleMacro((FunctionStyleMacro)expObject); + } else if (expObject instanceof ObjectStyleMacro) { + ObjectStyleMacro expMacro = (ObjectStyleMacro)expObject; + char[] expText = expMacro.expansion; + if (expText.length > 0) + pushContext(expText, expMacro); + } else if (expObject instanceof char[]) { + char[] expText = (char[])expObject; + if (expText.length > 0) + pushContext(expText); + } + return new MacroExpansionToken(); + } + + int tokenType = keywords.get(buffer, start, len); + if (tokenType < 0) + return new ImagedToken(IToken.tIDENTIFIER, new String(buffer, start, len)); + else + return new SimpleToken(tokenType); + } + + private IToken scanString() { + char[] buffer = bufferStack[bufferStackPos]; + + int tokenType = IToken.tSTRING; + if (buffer[bufferPos[bufferStackPos]] == 'L') { + ++bufferPos[bufferStackPos]; + tokenType = IToken.tLSTRING; + } + + int stringStart = bufferPos[bufferStackPos] + 1; + int stringLen = 0; + boolean escaped = false; + loop: + while (++bufferPos[bufferStackPos] < bufferLimit[bufferStackPos]) { + switch (buffer[bufferPos[bufferStackPos]]) { + case '\\': + ++stringLen; + escaped = !escaped; + continue; + case '"': + if (escaped) { + escaped = false; + continue; + } + break loop; + default: + ++stringLen; + escaped = false; + } + } + + // We should really throw an exception if we didn't get the terminating + // quote before the end of buffer + + return new ImagedToken(tokenType, new String(buffer, stringStart, stringLen)); + } + + private IToken scanNumber() { + char[] buffer = bufferStack[bufferStackPos]; + int start = bufferPos[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + + boolean isFloat = buffer[start] == '.'; + boolean hasExponent = false; + + boolean isHex = false; + if (buffer[start] == '0' && start + 1 < limit) { + switch (buffer[start + 1]) { + case 'x': + case 'X': + isHex = true; + ++bufferPos[bufferStackPos]; + } + } + + while (++bufferPos[bufferStackPos] < limit) { + int pos = bufferPos[bufferStackPos]; + switch (buffer[pos]) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + continue; + + case '.': + if (isFloat) + // second dot + break; + + isFloat = true; + continue; + + case 'E': + case 'e': + if (isHex) + // a hex 'e' + continue; + + if (hasExponent) + // second e + break; + + if (pos + 1 >= limit) + // ending on the e? + break; + + switch (buffer[pos + 1]) { + case '+': + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + // looks like a good exponent + isFloat = true; + hasExponent = true; + ++bufferPos[bufferStackPos]; + continue; + default: + // no exponent number? + break; + } + break; + + case 'a': + case 'A': + case 'b': + case 'B': + case 'c': + case 'C': + case 'd': + case 'D': + if (isHex) + continue; + + // not ours + --bufferPos[bufferStackPos]; + break; + + case 'f': + case 'F': + if (isHex) + continue; + + // must be float suffix + break; + + case 'p': + case 'P': + // Hex float exponent prefix + if (!isFloat || !isHex) { + --bufferPos[bufferStackPos]; + break; + } + + if (hasExponent) + // second p + break; + + if (pos + 1 >= limit) + // ending on the p? + break; + + switch (buffer[pos + 1]) { + case '+': + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + // looks like a good exponent + isFloat = true; + hasExponent = true; + ++bufferPos[bufferStackPos]; + continue; + default: + // no exponent number? + break; + } + break; + + case 'u': + case 'U': + // unsigned suffix + break; + + case 'l': + case 'L': + if (pos + 1 < limit) { + switch (buffer[pos + 1]) { + case 'l': + case 'L': + // long long + ++bufferPos[bufferStackPos]; + } + } + // long or long long + break; + + default: + // not part of our number + } + + // If we didn't continue in the switch, we're done + break; + } + + --bufferPos[bufferStackPos]; + + return new ImagedToken(isFloat ? IToken.tFLOATINGPT : IToken.tINTEGER, + new String(buffer, start, bufferPos[bufferStackPos] - start + 1)); + } + + private static char[] _define = "define".toCharArray(); + + + private void handlePPDirective() { + char[] buffer = bufferStack[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + + skipOverWhiteSpace(); + + // find the directive + int pos = ++bufferPos[bufferStackPos]; + + // if new line or end of buffer, we're done + if (pos >= limit || buffer[pos] == '\n') + return; + + // 'define' + if (pos + 5 < limit && CharArrayUtils.equals(buffer, pos, 6, _define)) { + bufferPos[bufferStackPos] += 5; + handlePPDefine(); + return; + } + + // don't know, chew up to the end of line and return + while (++bufferPos[bufferStackPos] < limit && buffer[bufferPos[bufferStackPos]] != '\n') + ; + } + + private void handlePPDefine() { + char[] buffer = bufferStack[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + + skipOverWhiteSpace(); + + // get the Identifier + int idstart = ++bufferPos[bufferStackPos]; + if (idstart >= limit) + return; + + char c = buffer[idstart]; + if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_')) { + skipToNewLine(); + return; + } + + int idlen = 1; + while (++bufferPos[bufferStackPos] < limit) { + c = buffer[bufferPos[bufferStackPos]]; + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || c == '_' || (c >= '0' && c <= '9')) { + ++idlen; + continue; + } else { + break; + } + } + --bufferPos[bufferStackPos]; + char[] name = new char[idlen]; + System.arraycopy(buffer, idstart, name, 0, idlen); + + // Now check for function style macro to store the arguments + char[][] arglist = null; + int pos = bufferPos[bufferStackPos]; + if (pos + 1 < limit && buffer[pos + 1] == '(') { + ++bufferPos[bufferStackPos]; + arglist = new char[4][]; + int currarg = -1; + while (bufferPos[bufferStackPos] < limit) { + skipOverWhiteSpace(); + if (++bufferPos[bufferStackPos] >= limit) + return; + c = buffer[bufferPos[bufferStackPos]]; + if (c == ')') { + break; + } else if (c == ',') { + continue; + } else if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + // yuck + skipToNewLine(); + return; + } + int argstart = bufferPos[bufferStackPos]; + skipOverIdentifier(); + if (++currarg == arglist.length) { + char[][] oldarglist = arglist; + arglist = new char[oldarglist.length * 2][]; + System.arraycopy(oldarglist, 0, arglist, 0, oldarglist.length); + } + int arglen = bufferPos[bufferStackPos] - argstart + 1; + char[] arg = new char[arglen]; + System.arraycopy(buffer, argstart, arg, 0, arglen); + arglist[currarg] = arg; + } + } + + // Capture the replacement text + skipOverWhiteSpace(); + int textstart = bufferPos[bufferStackPos] + 1; + int textend = textstart - 1; + + while (bufferPos[bufferStackPos] + 1 < limit + && buffer[bufferPos[bufferStackPos] + 1] != '\n') { + skipOverNonWhiteSpace(); + textend = bufferPos[bufferStackPos]; + skipOverWhiteSpace(); + } + + int textlen = textend - textstart + 1; + char[] text = emptyCharArray; + if (textlen > 0) { + text = new char[textlen]; + System.arraycopy(buffer, textstart, text, 0, textlen); + } + + // Throw it in + definitions.put(name, + arglist == null + ? new ObjectStyleMacro(name, text) + : new FunctionStyleMacro(name, text, arglist)); + } + + private void skipOverWhiteSpace() { + char[] buffer = bufferStack[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + + while (++bufferPos[bufferStackPos] < limit) { + int pos = bufferPos[bufferStackPos]; + switch (buffer[pos]) { + case ' ': + case '\t': + case '\r': + continue; + case '/': + if (pos + 1 < limit) { + if (buffer[pos + 1] == '/') { + // C++ comment, skip rest of line + skipToNewLine(); + return; + } else if (buffer[pos + 1] == '*') { + // C comment, find closing */ + for (bufferPos[bufferStackPos] += 2; + bufferPos[bufferStackPos] < limit; + ++bufferPos[bufferStackPos]) { + pos = bufferPos[bufferStackPos]; + if (buffer[pos] == '*' + && pos + 1 < limit + && buffer[pos + 1] == '/') { + ++bufferPos[bufferStackPos]; + break; + } + } + } + } + continue; + case '\\': + if (pos + 1 < limit && buffer[pos + 1] == '\n') { + // \n is a whitespace + ++bufferPos[bufferStackPos]; + continue; + } + } + + // fell out of switch without continuing, we're done + --bufferPos[bufferStackPos]; + return; + } + } + + private void skipOverNonWhiteSpace() { + char[] buffer = bufferStack[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + + while (++bufferPos[bufferStackPos] < limit) { + switch (buffer[bufferPos[bufferStackPos]]) { + case ' ': + case '\t': + case '\r': + case '\n': + --bufferPos[bufferStackPos]; + return; + case '\\': + int pos = bufferPos[bufferStackPos]; + if (pos + 1 < limit && buffer[pos + 1] == '\n') { + // \n is whitespace + --bufferPos[bufferStackPos]; + return; + } + break; + case '"': + boolean escaped = false; + loop: + while (++bufferPos[bufferStackPos] < bufferLimit[bufferStackPos]) { + switch (buffer[bufferPos[bufferStackPos]]) { + case '\\': + escaped = !escaped; + continue; + case '"': + if (escaped) { + escaped = false; + continue; + } + break loop; + default: + escaped = false; + } + } + + } + } + --bufferPos[bufferStackPos]; + } + + private void skipOverMacroArg() { + char[] buffer = bufferStack[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + + while (++bufferPos[bufferStackPos] < limit) { + switch (buffer[bufferPos[bufferStackPos]]) { + case ' ': + case '\t': + case '\r': + case '\n': + case ',': + case ')': + --bufferPos[bufferStackPos]; + return; + case '\\': + int pos = bufferPos[bufferStackPos]; + if (pos + 1 < limit && buffer[pos + 1] == '\n') { + // \n is whitespace + --bufferPos[bufferStackPos]; + return; + } + break; + case '"': + boolean escaped = false; + loop: + while (++bufferPos[bufferStackPos] < bufferLimit[bufferStackPos]) { + switch (buffer[bufferPos[bufferStackPos]]) { + case '\\': + escaped = !escaped; + continue; + case '"': + if (escaped) { + escaped = false; + continue; + } + break loop; + default: + escaped = false; + } + } + break; + } + } + --bufferPos[bufferStackPos]; + } + + private void skipOverIdentifier() { + char[] buffer = bufferStack[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + + while (++bufferPos[bufferStackPos] < limit) { + char c = buffer[bufferPos[bufferStackPos]]; + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || c == '_' || (c >= '0' && c <= '9')) { + continue; + } else { + break; + } + + } + --bufferPos[bufferStackPos]; + } + + private void skipToNewLine() { + char[] buffer = bufferStack[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + + int pos = bufferPos[bufferStackPos]; + if (pos >= limit || buffer[pos] == '\n') + return; + + boolean escaped = false; + while (++bufferPos[bufferStackPos] < limit) + switch (buffer[bufferPos[bufferStackPos]]) { + case '\\': + escaped = !escaped; + break; + case '\n': + if (escaped) { + escaped = false; + break; + } else { + return; + } + default: + escaped = false; + } + } + + private void handleFunctionStyleMacro(FunctionStyleMacro macro) { + char[] buffer = bufferStack[bufferStackPos]; + int limit = bufferLimit[bufferStackPos]; + + if (++bufferPos[bufferStackPos] >= limit + || buffer[bufferPos[bufferStackPos]] != '(') + return; + + FunctionStyleMacro.Expansion exp = macro.new Expansion(); + char[][] arglist = macro.arglist; + int currarg = -1; + int parens = 0; + + while (bufferPos[bufferStackPos] < limit) { + if (++currarg >= arglist.length || arglist[currarg] == null) + // too many args + break; + + skipOverWhiteSpace(); + + int pos = ++bufferPos[bufferStackPos]; + char c = buffer[pos]; + if (c == ')') { + if (parens == 0) + // end of macro + break; + else { + --parens; + continue; + } + } else if (c == ',') { + // empty arg + exp.definitions.put(arglist[currarg], emptyCharArray); + continue; + } else if (c == '(') { + ++parens; + continue; + } + + // peel off the arg + int argstart = pos; + int argend = argstart - 1; + + // Loop looking for end of argument + while (bufferPos[bufferStackPos] < limit) { + skipOverMacroArg(); + argend = bufferPos[bufferStackPos]; + skipOverWhiteSpace(); + + if (++bufferPos[bufferStackPos] >= limit) + break; + c = buffer[bufferPos[bufferStackPos]]; + if (c == ',' || c == ')') + break; + } + + char[] arg = emptyCharArray; + int arglen = argend - argstart + 1; + if (arglen > 0) { + arg = new char[arglen]; + System.arraycopy(buffer, argstart, arg, 0, arglen); + } + exp.definitions.put(arglist[currarg], arg); + + if (c == ')') + break; + } + + char[] expText = macro.expansion; + if (expText.length > 0) + pushContext(expText, exp); + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#nextToken(boolean) + */ + public IToken nextToken(boolean next) throws ScannerException, + EndOfFileException { + // TODO Auto-generated method stub + return null; + } + + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#nextTokenForStringizing() + */ + public IToken nextTokenForStringizing() throws ScannerException, + EndOfFileException { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#overwriteIncludePath(java.lang.String[]) + */ + public void overwriteIncludePath(String[] newIncludePaths) { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#setASTFactory(org.eclipse.cdt.core.parser.ast.IASTFactory) + */ + public void setASTFactory(IASTFactory f) { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#setOffsetBoundary(int) + */ + public void setOffsetBoundary(int offset) { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#setScannerContext(org.eclipse.cdt.internal.core.parser.scanner.IScannerContext) + */ + public void setScannerContext(IScannerContext context) { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#setThrowExceptionOnBadCharacterRead(boolean) + */ + public void setThrowExceptionOnBadCharacterRead(boolean throwOnBad) { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IScanner#setTokenizingMacroReplacementList(boolean) + */ + public void setTokenizingMacroReplacementList(boolean b) { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getASTFactory() + */ + public IASTFactory getASTFactory() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getBranchTracker() + */ + public BranchTracker getBranchTracker() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getClientRequestor() + */ + public ISourceElementRequestor getClientRequestor() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getContextStack() + */ + public ContextStack getContextStack() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getFileCache() + */ + public Map getFileCache() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getIncludePathNames() + */ + public List getIncludePathNames() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getInitialReader() + */ + public CodeReader getInitialReader() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getLanguage() + */ + public ParserLanguage getLanguage() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getLogService() + */ + public IParserLogService getLogService() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getOriginalConfig() + */ + public IScannerInfo getOriginalConfig() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getParserMode() + */ + public ParserMode getParserMode() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getPrivateDefinitions() + */ + public Map getPrivateDefinitions() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getProblemFactory() + */ + public IProblemFactory getProblemFactory() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getPublicDefinitions() + */ + public Map getPublicDefinitions() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getScanner() + */ + public IScanner getScanner() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#getWorkingCopies() + */ + public Iterator getWorkingCopies() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#parseInclusionDirective(java.lang.String, int) + */ + public InclusionDirective parseInclusionDirective(String restOfLine, + int offset) throws InclusionParseException { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#setDefinitions(java.util.Map) + */ + public void setDefinitions(Map map) { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) + * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerData#setIncludePathNames(java.util.List) + */ + public void setIncludePathNames(List includePathNames) { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IFilenameProvider#getCurrentFileIndex() + */ + public int getCurrentFileIndex() { + // TODO Auto-generated method stub + return 0; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IFilenameProvider#getCurrentFilename() + */ + public char[] getCurrentFilename() { + // TODO Auto-generated method stub + return null; + } + /* (non-Javadoc) + * @see org.eclipse.cdt.core.parser.IFilenameProvider#getFilenameForIndex(int) + */ + public String getFilenameForIndex(int index) { + // TODO Auto-generated method stub + return null; + } + + private static CharArrayIntMap keywords; + static { + keywords = new CharArrayIntMap(128); + + // Common keywords + keywords.put("auto".toCharArray(), IToken.t_auto); + keywords.put("break".toCharArray(), IToken.t_break); + keywords.put("case".toCharArray(), IToken.t_case); + keywords.put("char".toCharArray(), IToken.t_char); + keywords.put("const".toCharArray(), IToken.t_const); + keywords.put("continue".toCharArray(), IToken.t_continue); + keywords.put("default".toCharArray(), IToken.t_default); + keywords.put("do".toCharArray(), IToken.t_do); + keywords.put("double".toCharArray(), IToken.t_double); + keywords.put("else".toCharArray(), IToken.t_else); + keywords.put("enum".toCharArray(), IToken.t_enum); + keywords.put("extern".toCharArray(), IToken.t_extern); + keywords.put("float".toCharArray(), IToken.t_float); + keywords.put("for".toCharArray(), IToken.t_for); + keywords.put("goto".toCharArray(), IToken.t_goto); + keywords.put("if".toCharArray(), IToken.t_if); + keywords.put("inline".toCharArray(), IToken.t_inline); + keywords.put("int".toCharArray(), IToken.t_int); + keywords.put("long".toCharArray(), IToken.t_long); + keywords.put("register".toCharArray(), IToken.t_register); + keywords.put("return".toCharArray(), IToken.t_return); + keywords.put("short".toCharArray(), IToken.t_short); + keywords.put("signed".toCharArray(), IToken.t_signed); + keywords.put("sizeof".toCharArray(), IToken.t_sizeof); + keywords.put("static".toCharArray(), IToken.t_static); + keywords.put("struct".toCharArray(), IToken.t_struct); + keywords.put("switch".toCharArray(), IToken.t_switch); + keywords.put("typedef".toCharArray(), IToken.t_typedef); + keywords.put("union".toCharArray(), IToken.t_union); + keywords.put("unsigned".toCharArray(), IToken.t_unsigned); + keywords.put("void".toCharArray(), IToken.t_void); + keywords.put("volatile".toCharArray(), IToken.t_volatile); + keywords.put("while".toCharArray(), IToken.t_while); + + // ANSI C keywords + keywords.put("restrict".toCharArray(), IToken.t_restrict); + keywords.put("_Bool".toCharArray(), IToken.t__Bool); + keywords.put("_Complex".toCharArray(), IToken.t__Complex); + keywords.put("_Imaginary".toCharArray(), IToken.t__Imaginary); + + // C++ Keywords + keywords.put("asm".toCharArray(), IToken.t_asm); + keywords.put("bool".toCharArray(), IToken.t_bool); + keywords.put("catch".toCharArray(), IToken.t_catch); + keywords.put("class".toCharArray(), IToken.t_class); + keywords.put("const_cast".toCharArray(), IToken.t_const_cast); + keywords.put("delete".toCharArray(), IToken.t_delete); + keywords.put("dynamic_cast".toCharArray(), IToken.t_dynamic_cast); + keywords.put("explicit".toCharArray(), IToken.t_explicit); + keywords.put("export".toCharArray(), IToken.t_export); + keywords.put("false".toCharArray(), IToken.t_false); + keywords.put("friend".toCharArray(), IToken.t_friend); + keywords.put("mutable".toCharArray(), IToken.t_mutable); + keywords.put("namespace".toCharArray(), IToken.t_namespace); + keywords.put("new".toCharArray(), IToken.t_new); + keywords.put("operator".toCharArray(), IToken.t_operator); + keywords.put("private".toCharArray(), IToken.t_private); + keywords.put("protected".toCharArray(), IToken.t_protected); + keywords.put("public".toCharArray(), IToken.t_public); + keywords.put("reinterpret_cast".toCharArray(), IToken.t_reinterpret_cast); + keywords.put("static_cast".toCharArray(), IToken.t_static_cast); + keywords.put("template".toCharArray(), IToken.t_template); + keywords.put("this".toCharArray(), IToken.t_this); + keywords.put("throw".toCharArray(), IToken.t_throw); + keywords.put("true".toCharArray(), IToken.t_true); + keywords.put("try".toCharArray(), IToken.t_try); + keywords.put("typeid".toCharArray(), IToken.t_typeid); + keywords.put("typename".toCharArray(), IToken.t_typename); + keywords.put("using".toCharArray(), IToken.t_using); + keywords.put("virtual".toCharArray(), IToken.t_virtual); + keywords.put("wchar_t".toCharArray(), IToken.t_wchar_t); + + // C++ operator alternative + keywords.put("and".toCharArray(), IToken.t_and); + keywords.put("and_eq".toCharArray(), IToken.t_and_eq); + keywords.put("bitand".toCharArray(), IToken.t_bitand); + keywords.put("bitor".toCharArray(), IToken.t_bitor); + keywords.put("compl".toCharArray(), IToken.t_compl); + keywords.put("not".toCharArray(), IToken.t_not); + keywords.put("not_eq".toCharArray(), IToken.t_not_eq); + keywords.put("or".toCharArray(), IToken.t_or); + keywords.put("or_eq".toCharArray(), IToken.t_or_eq); + keywords.put("xor".toCharArray(), IToken.t_xor); + keywords.put("xor_eq".toCharArray(), IToken.t_xor_eq); + } +}