Patch for Bogdan Gheorghe:

- The initial framework for the new indexer.
- Added a checkbox to the Indexer tab on the
C/C++ Projects settings dialog to turn on the
indexing on a per project basis.

core/org.eclipse.cdt.core.tests/.classpath

@@ -6,6 +6,7 @@
     <classpathentry kind="src" path="parser/"/>
     <classpathentry kind="src" path="resources/"/>
     <classpathentry kind="src" path="suite/"/>
+    <classpathentry kind="src" path="indexer"/>
     <classpathentry kind="src" path="/org.apache.xerces"/>
     <classpathentry kind="src" path="/org.eclipse.core.resources"/>
     <classpathentry kind="src" path="/org.eclipse.core.runtime"/>

core/org.eclipse.cdt.core.tests/indexer/org/eclipse/cdt/core/indexer/tests/IndexManagerTest.java

@@ -0,0 +1,175 @@
+/**********************************************************************
+ * Copyright (c) 2002,2003 Rational Software Corporation and others.
+ * All rights reserved.   This program and the accompanying materials
+ * are made available under the terms of the Common Public License v0.5
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+***********************************************************************/
+/*
+ * Created on Jun 19, 2003
+ */
+package org.eclipse.cdt.core.indexer.tests;
+
+import java.io.FileInputStream;
+
+import org.eclipse.cdt.core.CCorePlugin;
+import org.eclipse.cdt.core.CProjectNature;
+import org.eclipse.cdt.internal.core.index.IEntryResult;
+import org.eclipse.cdt.internal.core.index.IIndex;
+import org.eclipse.cdt.internal.core.index.IQueryResult;
+import org.eclipse.cdt.internal.core.index.impl.IFileDocument;
+import org.eclipse.cdt.internal.core.search.indexing.IndexManager;
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IProject;
+import org.eclipse.core.resources.IProjectDescription;
+import org.eclipse.core.resources.IResource;
+import org.eclipse.core.resources.IWorkspace;
+import org.eclipse.core.resources.IWorkspaceRoot;
+import org.eclipse.core.resources.ResourcesPlugin;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.NullProgressMonitor;
+import org.eclipse.core.runtime.Path;
+import org.eclipse.core.runtime.Platform;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * @author bgheorgh
+ */
+public class IndexManagerTest extends TestCase {
+	IFile file;
+	IFileDocument fileDoc;
+	IProject testProject;
+	NullProgressMonitor monitor;
+    IndexManager indexManager;
+	/**
+	 * Constructor for IndexManagerTest.
+	 * @param name
+	 */
+	public IndexManagerTest(String name) {
+		super(name);
+	}
+
+	public static void main(String[] args) {
+	}
+	/*
+	 * @see TestCase#setUp()
+	 */
+	protected void setUp() throws Exception {
+		super.setUp();
+		//Create temp project
+		testProject = createProject("IndexerTestProject");
+		if (testProject==null)
+			fail("Unable to create project");
+		//Add a file to the project
+		importFile("mail.cpp","resources/indexer/mail.cpp");	
+		//Start up the index manager
+		//indexManager = new IndexManager();
+		//indexManager.reset();
+	}
+	/*
+	 * @see TestCase#tearDown()
+	 */
+	protected void tearDown() throws Exception {
+		super.tearDown();
+		//Delete project
+		//testProject.delete(true,monitor);
+	}
+
+	public static Test suite() { 
+		return new TestSuite(IndexManagerTest.class);
+	}
+	/*
+	 * Utils
+	 */
+	private IProject createProject(String projectName) throws CoreException
+	{
+	   IWorkspaceRoot root= ResourcesPlugin.getWorkspace().getRoot();
+	   IProject project= root.getProject(projectName);
+	   if (!project.exists()) {
+		 project.create(null);
+	   } else {
+	     project.refreshLocal(IResource.DEPTH_INFINITE, null);
+	   }
+	   if (!project.isOpen()) {
+		 project.open(null);
+	   }  
+	   if (!project.hasNature(CProjectNature.C_NATURE_ID)) {
+		   addNatureToProject(project, CProjectNature.C_NATURE_ID, null);
+	   }
+       //Fill out a project description
+	   IPath defaultPath = Platform.getLocation();
+	   IPath newPath = project.getFullPath();
+	   if (defaultPath.equals(newPath))
+		 newPath = null;
+	   IWorkspace workspace = ResourcesPlugin.getWorkspace();
+	   IProjectDescription description = workspace.newProjectDescription(project.getName());
+	   description.setLocation(newPath);
+	   //Create the project
+	   IProject cproject = CCorePlugin.getDefault().createCProject(description,project,monitor,CCorePlugin.PLUGIN_ID + ".make"); //.getCoreModel().create(project);
+	   
+	   return cproject; 
+	}
+	
+	private void importFile(String fileName, String resourceLocation)throws Exception{
+	   //Obtain file handle
+       file = testProject.getProject().getFile(fileName); 
+	   String pluginRoot=org.eclipse.core.runtime.Platform.getPlugin("org.eclipse.cdt.core.tests").find(new Path("/")).getFile();
+	   //Create file input stream
+	   monitor = new NullProgressMonitor();
+	   if (!file.exists()){
+		 file.create(new FileInputStream(pluginRoot + resourceLocation),false,monitor);
+	   }
+	   fileDoc = new IFileDocument(file);
+	}
+	
+	private void addNatureToProject(IProject proj, String natureId, IProgressMonitor monitor) throws CoreException {
+		   IProjectDescription description = proj.getDescription();
+		   String[] prevNatures= description.getNatureIds();
+		   String[] newNatures= new String[prevNatures.length + 1];
+		   System.arraycopy(prevNatures, 0, newNatures, 0, prevNatures.length);
+		   newNatures[prevNatures.length]= natureId;
+		   description.setNatureIds(newNatures);
+		   proj.setDescription(description, monitor);
+	 }
+	/*
+	 * Start of tests
+	 */ 	
+	public void testIndexAll() throws Exception {
+		//Enable indexing on the created project
+		//By doing this, we force the Index Manager to indexAll()
+		indexManager = CCorePlugin.getDefault().getCoreModel().getIndexManager();
+		indexManager.setEnabled(testProject,true);
+		
+		Thread.sleep(15000);
+		IIndex ind = indexManager.getIndex(testProject.getFullPath(),true,true);
+		char[] prefix = "typeDecl/".toCharArray();
+		IQueryResult[] qresults = ind.queryPrefix(prefix);
+		IEntryResult[] eresults = ind.queryEntries(prefix);
+		String [] queryResultModel = {"IndexedFile(1: /IndexerTestProject/mail.cpp)"};
+		String [] entryResultModel ={"EntryResult: word=typeDecl/C/Mail/, refs={ 1 }", "EntryResult: word=typeDecl/C/Unknown/, refs={ 1 }", "EntryResult: word=typeDecl/C/container/, refs={ 1 }", "EntryResult: word=typeDecl/C/first_class/, refs={ 1 }", "EntryResult: word=typeDecl/C/postcard/, refs={ 1 }"};
+		
+		if (qresults.length != queryResultModel.length)
+			fail("Query Result length different from model");
+
+		if (eresults.length != entryResultModel.length)
+			fail("Entry Result length different from model");
+		
+	
+		for (int i=0; i<qresults.length;i++)
+		{
+			assertEquals(queryResultModel[i],qresults[i].toString());
+		}
+		
+		for (int i=0;i<eresults.length; i++)
+		{
+			assertEquals(entryResultModel[i],eresults[i].toString());
+		}
+	}
+	
+	
+}

core/org.eclipse.cdt.core.tests/resources/indexer/mail.cpp

@@ -0,0 +1,98 @@
+#include <iostream.h>
+#include <stdlib.h>
+#include <alloc.h>
+#include <iomanip.h>
+
+class Mail
+{
+public:
+Mail(){}
+virtual void print()=0; //Pure Virtual Function, forces redefinition
+protected:
+float postage;
+char *type;
+friend ostream& operator << (ostream& os, Mail *m);
+};
+
+class postcard : public Mail
+{
+public:
+postcard(): Mail(){postage = 0.20; type = "Postcard";}
+void print(){cout << type << ": $" << setiosflags(ios::fixed)
+ <<setprecision(2) << postage <<endl;}
+ };
+
+class first_class : public Mail
+{
+public:
+first_class() : Mail(){postage = 0.32; type = "First Class";}
+void print(){cout << type << ": $" <<setiosflags(ios::fixed)
+             << setprecision(2) << postage <<endl;}
+ 
+};
+
+class Unknown : public postcard, first_class // ??? Multiple Inheritance
+{
+public:
+Unknown(): postcard(), first_class()
+{
+postcard::postage = 1.50; // MUST disambiguate
+postcard::type = "Unknown";
+}
+void print(){cout << postcard::type << ": $" <<setiosflags(ios::fixed)
+             <<setprecision(2)<<postcard::postage <<endl;}
+ };
+
+class container
+{
+private:
+Mail **array;
+int index;
+int sz;
+public:
+container(){array = 0;}
+~container(){
+for(int x = 0; x <sz; x++)
+ delete array[x];
+free(array);
+}
+int size() {return sz;}
+Mail* operator[](int index);
+Mail* operator = (Mail* mail);
+};
+
+main()
+{
+container PO_Box;
+PO_Box = new postcard;
+PO_Box = new first_class;
+PO_Box = new parcel_Post;
+//PO_Box = new Unknown;
+//one way of printing information
+for(int x =0; x <3; x++){
+ PO_Box[x]->print();
+}
+//Overloaded  <<
+ for(int x =0; x <PO_Box.size(); x++){
+ cout << PO_Box[x];
+ }
+}
+
+ostream& operator << (ostream &os, Mail *m)
+{
+os <<setiosflags(ios::fixed) << setprecision(2)<< m->type
+<< ": $" << m->postage <<endl;
+
+return os;
+}
+Mail* container::operator[](int index) {return array[index];}
+Mail* container::operator = (Mail* mail)
+{ 
+int size = sizeof(Mail*) * (++sz); 
+int temp = sz -1; 
+array = (Mail**)realloc(array, size); 
+array[temp] = mail; 
+return 0; 
+}
+
+

core/org.eclipse.cdt.core/ChangeLog

@@ -9,6 +9,17 @@
 	Added new constant for comma-separated lists
 	* src/org/eclipse/cdt/core/resources/IBuildInfo.java
 
+2003-06-25 Bogdan Gheorghe
+
+	* src/org/eclipse/cdt/core/CCorePlugin.java
+	Start the new indexer thread on startup
+	* src/org/eclipse/cdt/core/model/CoreModel.java
+	Added some methods to access the IndexManager
+	* src/org/eclipse/cdt/internal/core/model/CModelManager.java
+	Added some methods to access the IndexManager
+	* src/org/eclipse/cdt/internal/core/model/DeltaProcessor.java
+	Added IndexManager member
+
 2003-06-24 Alain Magloire
 
 	Patch form ando@park.ruru.ne.jp, to deal

core/org.eclipse.cdt.core/index/ChangeLog

@@ -0,0 +1,60 @@
+2003-06-25 Bogdan Gheorghe
+
+	Added new Indexer framework:
+	
+	* index/org/eclipse/cdt/internal/core/index/IDocument.java
+	* index/org/eclipse/cdt/internal/core/index/IEntryResult.java
+	* index/org/eclipse/cdt/internal/core/index/IIndex.java
+	* index/org/eclipse/cdt/internal/core/index/IIndexer.java
+	* index/org/eclipse/cdt/internal/core/index/IIndexerOutput.java
+	* index/org/eclipse/cdt/internal/core/index/IQueryResult.java
+	
+	* index/org/eclipse/cdt/internal/core/index/impl/Block.java
+	* index/org/eclipse/cdt/internal/core/index/impl/BlocksIndexInput.java
+	* index/org/eclipse/cdt/internal/core/index/impl/BlocksIndexOutput.java
+	* index/org/eclipse/cdt/internal/core/index/impl/CodeByteStream.java
+	* index/org/eclipse/cdt/internal/core/index/impl/EntryResult.java
+	* index/org/eclipse/cdt/internal/core/index/impl/Field.java
+	* index/org/eclipse/cdt/internal/core/index/impl/GammaCompressedIndexBlock.java
+	* index/org/eclipse/cdt/internal/core/index/impl/IFileDocument.java
+	* index/org/eclipse/cdt/internal/core/index/impl/IIndexConstants.java
+	* index/org/eclipse/cdt/internal/core/index/impl/Index.java
+	* index/org/eclipse/cdt/internal/core/index/impl/IndexBlock.java
+	* index/org/eclipse/cdt/internal/core/index/impl/IndexedFile.java
+	* index/org/eclipse/cdt/internal/core/index/impl/IndexedFileHashedArray.java
+	* index/org/eclipse/cdt/internal/core/index/impl/IndexerOutput.java
+	* index/org/eclipse/cdt/internal/core/index/impl/IndexInput.java
+	* index/org/eclipse/cdt/internal/core/index/impl/IndexOutput.java
+	* index/org/eclipse/cdt/internal/core/index/impl/IndexSummary.java
+	* index/org/eclipse/cdt/internal/core/index/impl/InMemoryIndex.java
+	* index/org/eclipse/cdt/internal/core/index/impl/Int.java
+	* index/org/eclipse/cdt/internal/core/index/impl/MergeFactory.java
+	* index/org/eclipse/cdt/internal/core/index/impl/PropertyDocument.java
+	* index/org/eclipse/cdt/internal/core/index/impl/SafeRandomAccessFile.java
+	* index/org/eclipse/cdt/internal/core/index/impl/SimpleIndexInput.java
+	* index/org/eclipse/cdt/internal/core/index/impl/Util.java
+	* index/org/eclipse/cdt/internal/core/index/impl/WordEntry.java
+	* index/org/eclipse/cdt/internal/core/index/impl/WordEntryHashedArray.java
+
+	* index/org/eclipse/cdt/internal/core/search/CharOperation.java
+	* index/org/eclipse/cdt/internal/core/search/HashtableOfInt.java
+	* index/org/eclipse/cdt/internal/core/search/SimpleLookupTable.java
+	* index/org/eclipse/cdt/internal/core/search/Util.java
+	* index/org/eclipse/cdt/internal/core/search/message.properties
+
+	* index/org/eclipse/cdt/internal/core/search/indexing/AbstractIndexer.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/AddCompilationUnitToIndex.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/AddFileToIndex.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/AddFolderToIndex.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/IIndexConstants.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/IndexAllProject.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/IndexManager.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/IndexRequest.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/ReadWriteMonitor.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/RemoveFolderFromIndex.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/RemoveFromIndex.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/SaveIndex.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/SourceIndexer.java
+	* index/org/eclipse/cdt/internal/core/search/indexing/SourceIndexerRequestor.java
+
+	

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/IDocument.java

@@ -0,0 +1,47 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.index;
+
+import java.io.IOException;
+
+/**
+ * An <code>IDocument</code> represent a data source, e.g.&nbsp;a <code>File</code> (<code>FileDocument</code>), 
+ * an <code>IFile</code> (<code>IFileDocument</code>), 
+ * or other kinds of data sources (URL, ...). An <code>IIndexer</code> indexes an<code>IDocument</code>.
+ */
+
+public interface IDocument {
+	/**
+	 * Returns the content of the document, in a byte array.
+	 */
+	byte[] getByteContent() throws IOException;
+	/**
+	 * Returns the content of the document, in a char array.
+	 */
+	char[] getCharContent() throws IOException;
+	/**
+	 * Returns the encoding for this document
+	 */
+	String getEncoding();
+	/**
+	 * returns the name of the document (e.g. its path for a <code>File</code>, or its relative path
+	 * in the workbench for an <code>IFile</code>).
+	 */
+	String getName();
+	/**
+	 * Returns the content of the document, as a String.
+	 */
+	public String getStringContent() throws IOException;
+	/**
+	 * Returns the type of the document.
+	 */
+	String getType();
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/IEntryResult.java

@@ -0,0 +1,17 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.index;
+
+public interface IEntryResult {
+	public int[] getFileReferences();
+	public char[] getWord();
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/IIndex.java

@@ -0,0 +1,75 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.index;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * An IIndex is the interface used to generate an index file, and to make queries on
+ * this index.
+ */
+
+public interface IIndex {
+	/**
+		 * Adds the given document to the index.
+		 */
+		void add(IDocument document, IIndexer indexer) throws IOException;
+		/**
+		 * Empties the index.
+		 */
+		void empty() throws IOException;
+		/**
+		 * Returns the index file on the disk.
+		 */
+		File getIndexFile();
+		/**
+		 * Returns the number of documents indexed.
+		 */
+		int getNumDocuments() throws IOException;
+		/**
+		 * Returns the number of unique words indexed.
+		 */
+		int getNumWords() throws IOException;
+		/**
+		 * Returns the path corresponding to a given document number
+		 */
+		String getPath(int documentNumber) throws IOException;
+		/**
+		 * Ansers true if has some changes to save.
+		 */
+		boolean hasChanged();
+		/**
+		 * Returns the paths of the documents containing the given word.
+		 */
+		IQueryResult[] query(String word) throws IOException;
+		/**
+		 * Returns all entries for a given word.
+		 */
+		IEntryResult[] queryEntries(char[] pattern) throws IOException;
+		/**
+		 * Returns the paths of the documents whose names contain the given word.
+		 */
+		IQueryResult[] queryInDocumentNames(String word) throws IOException;
+		/**
+		 * Returns the paths of the documents containing the given word prefix.
+		 */
+		IQueryResult[] queryPrefix(char[] prefix) throws IOException;
+		/**
+		 * Removes the corresponding document from the index.
+		 */
+		void remove(String documentName) throws IOException;
+		/**
+		 * Saves the index on the disk.
+		 */
+		void save() throws IOException;
+
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/IIndexer.java

@@ -0,0 +1,42 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.index;
+
+/**
+ * An <code>IIndexer</code> indexes ONE document at each time. It adds the document names and
+ * the words references to an IIndex. Each IIndexer can index certain types of document, and should
+ * not index the other files. 
+ */
+public interface IIndexer {
+	/**
+	 * Returns the file types the <code>IIndexer</code> handles.
+	 */
+
+	String[] getFileTypes();
+	/**
+	 * Indexes the given document, adding the document name and the word references 
+	 * to this document to the given <code>IIndex</code>.The caller should use 
+	 * <code>shouldIndex()</code> first to determine whether this indexer handles 
+	 * the given type of file, and only call this method if so. 
+	 */
+
+	void index(IDocument document, IIndexerOutput output) throws java.io.IOException;
+	/**
+	 * Sets the document types the <code>IIndexer</code> handles.
+	 */
+
+	public void setFileTypes(String[] fileTypes);
+	/**
+	 * Returns whether the <code>IIndexer</code> can index the given document or not.
+	 */
+
+	public boolean shouldIndex(IDocument document);
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/IIndexerOutput.java

@@ -0,0 +1,22 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.index;
+
+/**
+ * This class represents the output from an indexer to an index 
+ * for a single document.
+ */
+
+public interface IIndexerOutput {
+	public void addDocument(IDocument document);
+	public void addRef(char[] word);
+	public void addRef(String word);
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/IQueryResult.java

@@ -0,0 +1,15 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.index;
+
+public interface IQueryResult {
+	String getPath();
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/Block.java

@@ -0,0 +1,61 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+/**
+ * A block is a container that can hold information (a list of file names, a list of
+ * words, ...), be saved on the disk and loaded in memory.
+ */
+
+public abstract class Block {
+	/**
+	 * Size of the block
+	 */
+	protected int blockSize;
+
+	/**
+	 * Field in which the information is stored
+	 */
+	protected Field field;
+
+	public Block(int blockSize) {
+		this.blockSize= blockSize;
+		field= new Field(blockSize);
+	}
+	/**
+	 * Empties the block.
+	 */
+	public void clear() {
+		field.clear();
+	}
+	/**
+	 * Flushes the block
+	 */
+	public void flush() {
+	}
+	/**
+	 * Loads the block with the given number in memory, reading it from a RandomAccessFile.
+	 */
+	public void read(RandomAccessFile raf, int blockNum) throws IOException {
+		raf.seek(blockNum * (long) blockSize);
+		raf.readFully(field.buffer());
+	}
+	/**
+	 * Writes the block in a RandomAccessFile, giving it a block number.
+	 */
+	public void write(RandomAccessFile raf, int blockNum) throws IOException {
+		raf.seek(blockNum * (long) blockSize);
+		raf.write(field.buffer());
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/BlocksIndexInput.java

@@ -0,0 +1,391 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.ArrayList;
+
+import org.eclipse.cdt.internal.core.search.CharOperation;
+import org.eclipse.cdt.internal.core.index.IDocument;
+import org.eclipse.cdt.internal.core.index.IEntryResult;
+import org.eclipse.cdt.internal.core.index.IQueryResult;
+import org.eclipse.cdt.internal.core.search.Util;
+import org.eclipse.cdt.internal.core.util.LRUCache;
+
+/**
+ * This input is used for reading indexes saved using a BlocksIndexOutput.
+ */
+public class BlocksIndexInput extends IndexInput {
+	public static final int CACHE_SIZE= 16; // Cache 16 blocks of 8K each, for a cache size of 128K
+	protected FileListBlock currentFileListBlock;
+	protected int currentFileListBlockNum;
+	protected int currentIndexBlockNum;
+	protected IndexBlock currentIndexBlock;
+	private RandomAccessFile raf;
+	protected File indexFile;
+	protected LRUCache blockCache;
+	protected boolean opened= false;
+	protected IndexSummary summary;
+
+	public BlocksIndexInput(File inputFile) {
+		this.indexFile= inputFile;
+		blockCache= new LRUCache(CACHE_SIZE);
+	}
+	/**
+	 * @see IndexInput#clearCache()
+	 */
+	public void clearCache() {
+		blockCache= new LRUCache(CACHE_SIZE);
+	}
+	/**
+	 * @see IndexInput#close()
+	 */
+	public void close() throws IOException {
+		if (opened) {
+			summary= null;
+			opened= false;
+			if (raf != null)
+				raf.close();
+		}
+	}
+	/**
+	 * @see IndexInput#getCurrentFile()
+	 */
+	public IndexedFile getCurrentFile() throws IOException {
+		if (!hasMoreFiles())
+			return null;
+		IndexedFile file= null;
+		if ((file= currentFileListBlock.getFile(filePosition)) == null) {
+			currentFileListBlockNum= summary.getBlockNumForFileNum(filePosition);
+			currentFileListBlock= getFileListBlock(currentFileListBlockNum);
+			file= currentFileListBlock.getFile(filePosition);
+		}
+		return file;
+	}
+	/**
+	 * Returns the entry corresponding to the given word.
+	 */
+	protected WordEntry getEntry(char[] word) throws IOException {
+		int blockNum= summary.getBlockNumForWord(word);
+		if (blockNum == -1) return null;
+		IndexBlock block= getIndexBlock(blockNum);
+		return block.findExactEntry(word);
+	}
+	/**
+	 * Returns the FileListBlock with the given number.
+	 */
+	protected FileListBlock getFileListBlock(int blockNum) throws IOException {
+		Integer key= new Integer(blockNum);
+		Block block= (Block) blockCache.get(key);
+		if (block != null && block instanceof FileListBlock)
+			return (FileListBlock) block;
+		FileListBlock fileListBlock= new FileListBlock(IIndexConstants.BLOCK_SIZE);
+		fileListBlock.read(raf, blockNum);
+		blockCache.put(key, fileListBlock);
+		return fileListBlock;
+	}
+	/**
+	 * Returns the IndexBlock (containing words) with the given number.
+	 */
+	protected IndexBlock getIndexBlock(int blockNum) throws IOException {
+		Integer key= new Integer(blockNum);
+		Block block= (Block) blockCache.get(key);
+		if (block != null && block instanceof IndexBlock)
+			return (IndexBlock) block;
+		IndexBlock indexBlock= new GammaCompressedIndexBlock(IIndexConstants.BLOCK_SIZE);
+		indexBlock.read(raf, blockNum);
+		blockCache.put(key, indexBlock);
+		return indexBlock;
+	}
+	/**
+	 * @see IndexInput#getIndexedFile(int)
+	 */
+	public IndexedFile getIndexedFile(int fileNum) throws IOException {
+		int blockNum= summary.getBlockNumForFileNum(fileNum);
+		if (blockNum == -1)
+			return null;
+		FileListBlock block= getFileListBlock(blockNum);
+		return block.getFile(fileNum);
+	}
+	/**
+	 * @see IndexInput#getIndexedFile(IDocument)
+	 */
+	public IndexedFile getIndexedFile(IDocument document) throws java.io.IOException {
+		setFirstFile();
+		String name= document.getName();
+		while (hasMoreFiles()) {
+			IndexedFile file= getCurrentFile();
+			String path= file.getPath();
+			if (path.equals(name))
+				return file;
+			moveToNextFile();
+		}
+		return null;
+	}
+	/**
+	 * Returns the list of numbers of files containing the given word.
+	 */
+
+	protected int[] getMatchingFileNumbers(char[] word) throws IOException {
+		int blockNum= summary.getBlockNumForWord(word);
+		if (blockNum == -1)
+			return new int[0];
+		IndexBlock block= getIndexBlock(blockNum);
+		WordEntry entry= block.findExactEntry(word);
+		return entry == null ? new int[0] : entry.getRefs();
+	}
+	/**
+	 * @see IndexInput#getNumFiles()
+	 */
+	public int getNumFiles() {
+		return summary.getNumFiles();
+	}
+	/**
+	 * @see IndexInput#getNumWords()
+	 */
+	public int getNumWords() {
+		return summary.getNumWords();
+	}
+	/**
+	 * @see IndexInput#getSource()
+	 */
+	public Object getSource() {
+		return indexFile;
+	}
+	/**
+	 * Initialises the blocksIndexInput
+	 */
+	protected void init() throws IOException {
+		clearCache();
+		setFirstFile();
+		setFirstWord();
+	}
+	/**
+	 * @see IndexInput#moveToNextFile()
+	 */
+	public void moveToNextFile() throws IOException {
+		filePosition++;
+	}
+	/**
+	 * @see IndexInput#moveToNextWordEntry()
+	 */
+	public void moveToNextWordEntry() throws IOException {
+		wordPosition++;
+		if (!hasMoreWords()) {
+			return;
+		}
+		//if end of the current block, we load the next one.
+		boolean endOfBlock= !currentIndexBlock.nextEntry(currentWordEntry);
+		if (endOfBlock) {
+			currentIndexBlock= getIndexBlock(++currentIndexBlockNum);
+			currentIndexBlock.nextEntry(currentWordEntry);
+		}
+	}
+	/**
+	 * @see IndexInput#open()
+	 */
+
+	public void open() throws IOException {
+		if (!opened) {
+			raf= new SafeRandomAccessFile(indexFile, "r"); //$NON-NLS-1$
+			String sig= raf.readUTF();
+			if (!sig.equals(IIndexConstants.SIGNATURE))
+				throw new IOException(Util.bind("exception.wrongFormat")); //$NON-NLS-1$
+			int summaryBlockNum= raf.readInt();
+			raf.seek(summaryBlockNum * (long) IIndexConstants.BLOCK_SIZE);
+			summary= new IndexSummary();
+			summary.read(raf);
+			init();
+			opened= true;
+		}
+	}
+	/**
+	 * @see IndexInput#query(String)
+	 */
+	public IQueryResult[] query(String word) throws IOException {
+		open();
+		int[] fileNums= getMatchingFileNumbers(word.toCharArray());
+		int size= fileNums.length;
+		IQueryResult[] files= new IQueryResult[size];
+		for (int i= 0; i < size; ++i) {
+			files[i]= getIndexedFile(fileNums[i]);
+		}
+		return files;
+	}
+	/**
+	 * If no prefix is provided in the pattern, then this operation will have to walk
+	 * all the entries of the whole index.
+	 */
+	public IEntryResult[] queryEntriesMatching(char[] pattern/*, boolean isCaseSensitive*/) throws IOException {
+		open();
+	
+		if (pattern == null || pattern.length == 0) return null;
+		int[] blockNums = null;
+		int firstStar = CharOperation.indexOf('*', pattern);
+		switch (firstStar){
+			case -1 :
+				WordEntry entry = getEntry(pattern);
+				if (entry == null) return null;
+				return new IEntryResult[]{ new EntryResult(entry.getWord(), entry.getRefs()) };
+			case 0 :
+				blockNums = summary.getAllBlockNums();
+				break;
+			default :
+				char[] prefix = CharOperation.subarray(pattern, 0, firstStar);
+				blockNums = summary.getBlockNumsForPrefix(prefix);
+		}
+		if (blockNums == null || blockNums.length == 0)	return null;
+				
+		IEntryResult[] entries = new IEntryResult[5];
+		int count = 0;
+		for (int i = 0, max = blockNums.length; i < max; i++) {
+			IndexBlock block = getIndexBlock(blockNums[i]);
+			block.reset();
+			boolean found = false;
+			WordEntry entry = new WordEntry();
+			while (block.nextEntry(entry)) {
+				if (CharOperation.match(entry.getWord(), pattern, true)) {
+					if (count == entries.length){
+						System.arraycopy(entries, 0, entries = new IEntryResult[count*2], 0, count);
+					}
+					entries[count++] = new EntryResult(entry.getWord(), entry.getRefs());
+					found = true;
+				} else {
+					if (found) break;
+				}
+			}
+		}
+		if (count != entries.length){
+			System.arraycopy(entries, 0, entries = new IEntryResult[count], 0, count);
+		}
+		return entries;
+	}
+	public IEntryResult[] queryEntriesPrefixedBy(char[] prefix) throws IOException {
+		open();
+		
+		int blockLoc = summary.getFirstBlockLocationForPrefix(prefix);
+		if (blockLoc < 0) return null;
+			
+		IEntryResult[] entries = new IEntryResult[5];
+		int count = 0;
+		while(blockLoc >= 0){
+			IndexBlock block = getIndexBlock(summary.getBlockNum(blockLoc));
+			block.reset();
+			boolean found = false;
+			WordEntry entry = new WordEntry();
+			while (block.nextEntry(entry)) {
+				if (CharOperation.prefixEquals(prefix, entry.getWord())) {
+					if (count == entries.length){
+						System.arraycopy(entries, 0, entries = new IEntryResult[count*2], 0, count);
+					}
+					entries[count++] = new EntryResult(entry.getWord(), entry.getRefs());
+					found = true;
+				} else {
+					if (found) break;
+				}
+			}
+			/* consider next block ? */
+			blockLoc = summary.getNextBlockLocationForPrefix(prefix, blockLoc);				
+		}
+		if (count == 0) return null;
+		if (count != entries.length){
+			System.arraycopy(entries, 0, entries = new IEntryResult[count], 0, count);
+		}
+		return entries;
+	}
+	public IQueryResult[] queryFilesReferringToPrefix(char[] prefix) throws IOException {
+		open();
+		
+		int blockLoc = summary.getFirstBlockLocationForPrefix(prefix);
+		if (blockLoc < 0) return null;
+			
+		// each filename must be returned already once
+		org.eclipse.cdt.internal.core.search.HashtableOfInt fileMatches = new  org.eclipse.cdt.internal.core.search.HashtableOfInt(20);
+		int count = 0; 
+		while(blockLoc >= 0){
+			IndexBlock block = getIndexBlock(summary.getBlockNum(blockLoc));
+			block.reset();
+			boolean found = false;
+			WordEntry entry = new WordEntry();
+			while (block.nextEntry(entry)) {
+				if (CharOperation.prefixEquals(prefix, entry.getWord()/*, isCaseSensitive*/)) {
+					int [] refs = entry.getRefs();
+					for (int i = 0, max = refs.length; i < max; i++){
+						int ref = refs[i];
+						if (!fileMatches.containsKey(ref)){
+							count++;
+							fileMatches.put(ref, getIndexedFile(ref));
+						}
+					}
+					found = true;
+				} else {
+					if (found) break;
+				}
+			}
+			/* consider next block ? */
+			blockLoc = summary.getNextBlockLocationForPrefix(prefix, blockLoc);				
+		}
+		/* extract indexed files */
+		IQueryResult[] files = new IQueryResult[count];
+		Object[] indexedFiles = fileMatches.valueTable;
+		for (int i = 0, index = 0, max = indexedFiles.length; i < max; i++){
+			IndexedFile indexedFile = (IndexedFile) indexedFiles[i];
+			if (indexedFile != null){
+				files[index++] = indexedFile;
+			}
+		}	
+		return files;
+	}
+	/**
+	 * @see IndexInput#queryInDocumentNames(String)
+	 */
+	public IQueryResult[] queryInDocumentNames(String word) throws IOException {
+		open();
+		ArrayList matches= new ArrayList();
+		setFirstFile();
+		while (hasMoreFiles()) {
+			IndexedFile file= getCurrentFile();
+			if (file.getPath().indexOf(word) != -1)
+				matches.add(file);
+			moveToNextFile();
+		}
+		IQueryResult[] match= new IQueryResult[matches.size()];
+		matches.toArray(match);
+		return match;
+	}
+	/**
+	 * @see IndexInput#setFirstFile()
+	 */
+
+	protected void setFirstFile() throws IOException {
+		filePosition= 1;
+		if (getNumFiles() > 0) {
+			currentFileListBlockNum= summary.getBlockNumForFileNum(1);
+			currentFileListBlock= getFileListBlock(currentFileListBlockNum);
+		}
+	}
+	/**
+	 * @see IndexInput#setFirstWord()
+	 */
+
+	protected void setFirstWord() throws IOException {
+		wordPosition= 1;
+		if (getNumWords() > 0) {
+			currentIndexBlockNum= summary.getFirstWordBlockNum();
+			currentIndexBlock= getIndexBlock(currentIndexBlockNum);
+			currentWordEntry= new WordEntry();
+			currentIndexBlock.reset();
+			currentIndexBlock.nextEntry(currentWordEntry);
+		}
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/BlocksIndexOutput.java

@@ -0,0 +1,167 @@
+ /*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+/**
+ * A blocksIndexOutput is used to save an index in a file with the given structure:<br>
+ *  - Signature of the file;<br>
+ *  - FileListBlocks;<br>
+ *  - IndexBlocks;<br>
+ *  - Summary of the index.
+ */
+
+public class BlocksIndexOutput extends IndexOutput {
+	protected RandomAccessFile indexOut;
+	protected int blockNum;
+	protected boolean opened= false;
+	protected File indexFile;
+	protected FileListBlock fileListBlock;
+	protected IndexBlock indexBlock;
+	protected int numWords= 0;
+	protected IndexSummary summary;
+	protected int numFiles= 0;
+	protected boolean firstInBlock;
+	protected boolean firstIndexBlock;
+	protected boolean firstFileListBlock;
+
+	public BlocksIndexOutput(File indexFile) {
+		this.indexFile= indexFile;
+		summary= new IndexSummary();
+		blockNum= 1;
+		firstInBlock= true;
+		firstIndexBlock= true;
+		firstFileListBlock= true;
+	}
+	/**
+	 * @see IndexOutput#addFile
+	 */
+	public void addFile(IndexedFile indexedFile) throws IOException {
+		if (firstFileListBlock) {
+			firstInBlock= true;
+			fileListBlock= new FileListBlock(IIndexConstants.BLOCK_SIZE);
+			firstFileListBlock= false;
+		}
+		if (fileListBlock.addFile(indexedFile)) {
+			if (firstInBlock) {
+				summary.addFirstFileInBlock(indexedFile, blockNum);
+				firstInBlock= false;
+			}
+			numFiles++;
+		} else {
+			if (fileListBlock.isEmpty()) {
+				return;
+			}
+			flushFiles();
+			addFile(indexedFile);
+		}
+	}
+	/**
+	 * @see IndexOutput#addWord
+	 */
+	public void addWord(WordEntry entry) throws IOException {
+		if (firstIndexBlock) {
+			indexBlock= new GammaCompressedIndexBlock(IIndexConstants.BLOCK_SIZE);
+			firstInBlock= true;
+			firstIndexBlock= false;
+		}
+		if (entry.getNumRefs() == 0)
+			return;
+		if (indexBlock.addEntry(entry)) {
+			if (firstInBlock) {
+				summary.addFirstWordInBlock(entry.getWord(), blockNum);
+				firstInBlock= false;
+			}
+			numWords++;
+		} else {
+			if (indexBlock.isEmpty()) {
+				return;
+			}
+			flushWords();
+			addWord(entry);
+		}
+	}
+	/**
+	 * @see IndexOutput#close
+	 */
+	public void close() throws IOException {
+		if (opened) {
+			indexOut.close();
+			summary= null;
+			numFiles= 0;
+			opened= false;
+		}
+	}
+	/**
+	 * @see IndexOutput#flush
+	 */
+	public void flush() throws IOException {
+		
+		summary.setNumFiles(numFiles);
+		summary.setNumWords(numWords);
+		indexOut.seek(blockNum * (long) IIndexConstants.BLOCK_SIZE);
+		summary.write(indexOut);
+		indexOut.seek(0);
+		indexOut.writeUTF(IIndexConstants.SIGNATURE);
+		indexOut.writeInt(blockNum);
+	}
+	/**
+	 * Writes the current fileListBlock on the disk and initialises it
+	 * (when it's full or it's the end of the index).
+	 */
+	protected void flushFiles() throws IOException {
+		if (!firstFileListBlock
+				&& fileListBlock != null) {
+			fileListBlock.flush();
+			fileListBlock.write(indexOut, blockNum++);
+			fileListBlock.clear();
+			firstInBlock= true;
+		}
+	}
+	/**
+	 * Writes the current indexBlock on the disk and initialises it
+	 * (when it's full or it's the end of the index).
+	 */
+	protected void flushWords() throws IOException {
+		if (!firstInBlock 
+				&& indexBlock != null) { // could have added a document without any indexed word, no block created yet
+			indexBlock.flush();
+			indexBlock.write(indexOut, blockNum++);
+			indexBlock.clear();
+			firstInBlock= true;
+		}
+	}
+	/**
+	 * @see IndexOutput#getDestination
+	 */
+	public Object getDestination() {
+		return indexFile;
+	}
+	/**
+	 * @see IndexOutput#open
+	 */
+	public void open() throws IOException {
+		if (!opened) {
+			summary= new IndexSummary();
+			numFiles= 0;
+			numWords= 0;
+			blockNum= 1;
+			firstInBlock= true;
+			firstIndexBlock= true;
+			firstFileListBlock= true;
+			indexOut= new SafeRandomAccessFile(this.indexFile, "rw"); //$NON-NLS-1$
+			opened= true;
+		}
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/CodeByteStream.java

@@ -0,0 +1,343 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.UTFDataFormatException;
+
+public class CodeByteStream {
+	protected byte[] bytes;
+	protected int byteOffset= 0;
+	protected int bitOffset= 0;
+	protected int markByteOffset= -1;
+	protected int markBitOffset= -1;
+
+	public CodeByteStream() {
+		this(16);
+	}
+	public CodeByteStream(byte[] bytes) {
+		this.bytes= bytes;
+	}
+	public CodeByteStream(int initialByteLength) {
+		bytes= new byte[initialByteLength];
+	}
+	public int byteLength() {
+		return (bitOffset + 7) / 8 + byteOffset;
+	}
+	public byte[] getBytes(int startOffset, int endOffset) {
+		int byteLength= byteLength();
+		if (startOffset > byteLength || endOffset > byteLength || startOffset > endOffset)
+			throw new IndexOutOfBoundsException();
+		int length= endOffset - startOffset;
+		byte[] result= new byte[length];
+		System.arraycopy(bytes, startOffset, result, 0, length);
+		if (endOffset == byteLength && bitOffset != 0) {
+			int mask= (1 << bitOffset) - 1;
+			result[length - 1] &= (mask << 8 - bitOffset);
+		}
+		return result;
+	}
+	protected void grow() {
+		byte[] newBytes= new byte[bytes.length * 2 + 1];
+		System.arraycopy(bytes, 0, newBytes, 0, bytes.length);
+		bytes= newBytes;
+	}
+	public void mark() {
+		markByteOffset= byteOffset;
+		markBitOffset= bitOffset;
+	}
+	/**
+	 * Reads a single bit (value == 0 or == 1).
+	 */
+	public int readBit() {
+		int value= (bytes[byteOffset] >> (7 - bitOffset)) & 1;
+		if (++bitOffset >= 8) {
+			bitOffset= 0;
+			++byteOffset;
+		}
+		return value;
+	}
+	/**
+	 * Read up to 32 bits from the stream.
+	 */
+	public int readBits(int numBits) {
+		int value= 0;
+		while (numBits > 0) {
+			int bitsToRead= 8 - bitOffset;
+			if (bitsToRead > numBits)
+				bitsToRead= numBits;
+			int mask= (1 << bitsToRead) - 1;
+			value |= ((bytes[byteOffset] >> (8 - bitOffset - bitsToRead)) & mask) << (numBits - bitsToRead);
+			numBits -= bitsToRead;
+			bitOffset += bitsToRead;
+			if (bitOffset >= 8) {
+				bitOffset -= 8;
+				byteOffset += 1;
+			}
+		}
+		return value;
+	}
+	public final int readByte() {
+
+		// no need to rebuild byte value from bit sequences
+		if (bitOffset == 0) return bytes[byteOffset++] & 255;
+	
+		int value= 0;
+		int numBits = 8;
+		while (numBits > 0) {
+			int bitsToRead= 8 - bitOffset;
+			if (bitsToRead > numBits)
+				bitsToRead= numBits;
+			int mask= (1 << bitsToRead) - 1;
+			value |= ((bytes[byteOffset] >> (8 - bitOffset - bitsToRead)) & mask) << (numBits - bitsToRead);
+			numBits -= bitsToRead;
+			bitOffset += bitsToRead;
+			if (bitOffset >= 8) {
+				bitOffset -= 8;
+				byteOffset += 1;
+			}
+		}
+		return value;
+	}
+	/**
+	 * Reads a value using Gamma coding.
+	 */
+	public int readGamma() {
+		int numBits= readUnary();
+		return readBits(numBits - 1) | (1 << (numBits - 1));
+	}
+	public char[] readUTF() throws UTFDataFormatException {
+		int utflen= readByte();
+		if (utflen == 255) {
+			// long UTF
+			int high = readByte();
+			int low = readByte();
+			utflen = (high << 8) + low;
+		}
+		char str[]= new char[utflen];
+		int count= 0;
+		int strlen= 0;
+		while (count < utflen) {
+			int c= readByte();
+			int char2, char3;
+			switch (c >> 4) {
+				case 0 :
+				case 1 :
+				case 2 :
+				case 3 :
+				case 4 :
+				case 5 :
+				case 6 :
+				case 7 :
+					// 0xxxxxxx
+					count++;
+					str[strlen++]= (char) c;
+					break;
+				case 12 :
+				case 13 :
+					// 110x xxxx   10xx xxxx
+					count += 2;
+					if (count > utflen)
+						throw new UTFDataFormatException();
+					char2= readByte();
+					if ((char2 & 0xC0) != 0x80)
+						throw new UTFDataFormatException();
+					str[strlen++]= (char) (((c & 0x1F) << 6) | (char2 & 0x3F));
+					break;
+				case 14 :
+					// 1110 xxxx  10xx xxxx  10xx xxxx
+					count += 3;
+					if (count > utflen)
+						throw new UTFDataFormatException();
+					char2= readByte();
+					char3= readByte();
+					if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80))
+						throw new UTFDataFormatException();
+					str[strlen++]= (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | ((char3 & 0x3F) << 0));
+					break;
+				default :
+					// 10xx xxxx,  1111 xxxx
+					throw new UTFDataFormatException();
+			}
+		}
+		if (strlen < utflen)
+			System.arraycopy(str, 0, str= new char[strlen], 0, strlen);
+		return str;
+	}
+	/**
+	 *  Reads a value in unary.
+	 */
+	public int readUnary() {
+		int value= 1;
+		int mask= 1 << (7 - bitOffset);
+		while ((bytes[byteOffset] & mask) != 0) {
+			++value;
+			if (++bitOffset >= 8) {
+				bitOffset= 0;
+				++byteOffset;
+				mask= 0x80;
+			} else {
+				mask >>>= 1;
+			}
+		}
+		// skip the 0 bit
+		if (++bitOffset >= 8) {
+			bitOffset= 0;
+			++byteOffset;
+		}
+		return value;
+	}
+	public void reset() {
+		byteOffset= bitOffset= 0;
+		markByteOffset= markBitOffset= -1;
+	}
+	public void reset(byte[] bytes) {
+		this.bytes= bytes;
+		reset();
+	}
+	public void reset(byte[] bytes, int byteOffset) {
+		reset(bytes);
+		this.byteOffset= byteOffset;
+	}
+	public boolean resetToMark() {
+		if (markByteOffset == -1)
+			return false;
+		byteOffset= markByteOffset;
+		bitOffset= markBitOffset;
+		markByteOffset= markBitOffset= -1;
+		return true;
+	}
+	public void skipBits(int numBits) {
+		int newOffset= byteOffset * 8 + bitOffset + numBits;
+		if (newOffset < 0 || (newOffset + 7) / 8 >= bytes.length)
+			throw new IllegalArgumentException();
+		byteOffset= newOffset / 8;
+		bitOffset= newOffset % 8;
+	}
+	public byte[] toByteArray() {
+		return getBytes(0, byteLength());
+	}
+	/**
+	 * Writes a single bit (value == 0 or == 1).
+	 */
+	public void writeBit(int value) {
+		bytes[byteOffset] |= (value & 1) << (7 - bitOffset);
+		if (++bitOffset >= 8) {
+			bitOffset= 0;
+			if (++byteOffset >= bytes.length)
+				grow();
+		}
+	}
+	/**
+	 * Write up to 32 bits to the stream.
+	 * The least significant numBits bits of value are written.
+	 */
+	public void writeBits(int value, int numBits) {
+		while (numBits > 0) {
+			int bitsToWrite= 8 - bitOffset;
+			if (bitsToWrite > numBits)
+				bitsToWrite= numBits;
+			int shift= 8 - bitOffset - bitsToWrite;
+			int mask= ((1 << bitsToWrite) - 1) << shift;
+			bytes[byteOffset]= (byte) ((bytes[byteOffset] & ~mask) | (((value >>> (numBits - bitsToWrite)) << shift) & mask));
+			numBits -= bitsToWrite;
+			bitOffset += bitsToWrite;
+			if (bitOffset >= 8) {
+				bitOffset -= 8;
+				if (++byteOffset >= bytes.length)
+					grow();
+			}
+		}
+	}
+	public void writeByte(int value) {
+		writeBits(value, 8);
+	}
+	/**
+	 * Writes the given value using Gamma coding, in which positive integer x
+	 * is represented by coding floor(log2(x) in unary followed by the value 
+	 * of x - 2**floor(log2(x)) in binary.
+	 * The value must be >= 1.
+	 */
+	public void writeGamma(int value) {
+		if (value < 1)
+			throw new IllegalArgumentException();
+		int temp= value;
+		int numBits= 0;
+		while (temp != 0) {
+			temp >>>= 1;
+			++numBits;
+		}
+		writeUnary(numBits);
+		writeBits(value, numBits - 1);
+	}
+	public void writeUTF(char[] str, int start, int end) {
+		int utflen= 0;
+		for (int i= start; i < end; i++) {
+			int c= str[i];
+			if ((c >= 0x0001) && (c <= 0x007F)) {
+				utflen++;
+			} else if (c > 0x07FF) {
+				utflen += 3;
+			} else {
+				utflen += 2;
+			}
+		}
+		if (utflen < 255) {
+			writeByte(utflen & 0xFF);
+		} else if (utflen > 65535) {
+			throw new IllegalArgumentException();
+		} else {
+			writeByte(255); // marker for long UTF
+			writeByte((utflen >>> 8) & 0xFF); // high byte
+			writeByte((utflen >>> 0) & 0xFF); // low byte
+		}
+		for (int i= start; i < end; i++) {
+			int c= str[i];
+			if ((c >= 0x0001) && (c <= 0x007F)) {
+				writeByte(c);
+			} else if (c > 0x07FF) {
+				writeByte(0xE0 | ((c >> 12) & 0x0F));
+				writeByte(0x80 | ((c >> 6) & 0x3F));
+				writeByte(0x80 | ((c >> 0) & 0x3F));
+			} else {
+				writeByte(0xC0 | ((c >> 6) & 0x1F));
+				writeByte(0x80 | ((c >> 0) & 0x3F));
+			}
+		}
+	}
+	/**
+	 * Write the given value in unary.  The value must be >= 1.
+	 */
+	public void writeUnary(int value) {
+		if (value < 1)
+			throw new IllegalArgumentException();
+		int mask= 1 << (7 - bitOffset);
+		// write N-1 1-bits
+		while (--value > 0) {
+			bytes[byteOffset] |= mask;
+			if (++bitOffset >= 8) {
+				bitOffset= 0;
+				if (++byteOffset >= bytes.length)
+					grow();
+				mask= 0x80;
+			} else {
+				mask >>>= 1;
+			}
+		}
+		// write a 0-bit
+		bytes[byteOffset] &= ~mask;
+		if (++bitOffset >= 8) {
+			bitOffset= 0;
+			if (++byteOffset >= bytes.length)
+				grow();
+		}
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/EntryResult.java

@@ -0,0 +1,68 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import org.eclipse.cdt.internal.core.index.IEntryResult;
+import org.eclipse.cdt.internal.core.search.CharOperation;
+
+
+public class EntryResult implements IEntryResult {
+	private char[] word;
+	private int[]  fileRefs;
+	
+public EntryResult(char[] word, int[] refs) {
+	this.word = word;
+	this.fileRefs = refs;
+}
+public boolean equals(Object anObject){
+	
+	if (this == anObject) {
+		return true;
+	}
+	if ((anObject != null) && (anObject instanceof EntryResult)) {
+		EntryResult anEntryResult = (EntryResult) anObject;
+		if (!CharOperation.equals(this.word, anEntryResult.word)) return false;
+
+		int length;
+		int[] refs, otherRefs;
+		if ((length = (refs = this.fileRefs).length) != (otherRefs = anEntryResult.fileRefs).length) return false;
+		for (int i =  0; i < length; i++){
+			if (refs[i] != otherRefs[i]) return false;
+		}
+		return true;
+	}
+	return false;
+	
+}
+public int[] getFileReferences() {
+	return fileRefs;
+}
+public char[] getWord() {
+	return word;
+}
+public int hashCode(){
+	return CharOperation.hashCode(word);
+}
+public String toString(){
+	StringBuffer buffer = new StringBuffer(word.length * 2);
+	buffer.append("EntryResult: word="); //$NON-NLS-1$
+	buffer.append(word);
+	buffer.append(", refs={"); //$NON-NLS-1$
+	for (int i = 0; i < fileRefs.length; i++){
+		if (i > 0) buffer.append(',');
+		buffer.append(' ');
+		buffer.append(fileRefs[i]);
+	}
+	buffer.append(" }"); //$NON-NLS-1$
+	return buffer.toString();
+}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/Field.java

@@ -0,0 +1,382 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+ 
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.UTFDataFormatException;
+
+public class Field {
+	protected byte[] buffer; // contents
+	protected int offset; // offset of the field within the byte array
+	protected int length; // length of the field
+
+	/**
+	 * ByteSegment constructor comment.
+	 */
+	public Field(byte[] bytes) {
+		this.buffer= bytes;
+		this.offset= 0;
+		this.length= bytes.length;
+	}
+	/**
+	 * ByteSegment constructor comment.
+	 */
+	public Field(byte[] bytes, int length) {
+		this.buffer= bytes;
+		this.offset= 0;
+		this.length= length;
+	}
+	/**
+	 * ByteSegment constructor comment.
+	 */
+	public Field(byte[] bytes, int offset, int length) {
+		this.buffer= bytes;
+		this.offset= offset;
+		this.length= length;
+	}
+	/**
+	 * Creates a new field containing an empty buffer of the given length.
+	 */
+	public Field(int length) {
+		this.buffer= new byte[length];
+		this.offset= 0;
+		this.length= length;
+	}
+	public byte[] buffer() {
+		return buffer;
+	}
+	public Field buffer(byte[] buffer) {
+		this.buffer= buffer;
+		return this;
+	}
+	public Field clear() {
+		clear(buffer, offset, length);
+		return this;
+	}
+	protected static void clear(byte[] buffer, int offset, int length) {
+		int n= offset;
+		for (int i= 0; i < length; i++) {
+			buffer[n]= 0;
+			n++;
+		}
+	}
+	public Field clear(int length) {
+		clear(buffer, offset, length);
+		return this;
+	}
+	public Field clear(int offset, int length) {
+		clear(buffer, this.offset + offset, length);
+		return this;
+	}
+	protected static int compare(byte[] buffer1, int offset1, int length1, byte[] buffer2, int offset2, int length2) {
+		int n= Math.min(length1, length2);
+		for (int i= 0; i < n; i++) {
+			int j1= buffer1[offset1 + i] & 255;
+			int j2= buffer2[offset2 + i] & 255;
+			if (j1 > j2)
+				return 1;
+			if (j1 < j2)
+				return -1;
+		}
+		if (length1 > n) {
+			for (int i= n; i < length1; i++)
+				if (buffer1[offset1 + i] != 0)
+					return 1;
+			return 0;
+		}
+		for (int i= n; i < length2; i++)
+			if (buffer2[offset2 + i] != 0)
+				return -1;
+		return 0;
+	}
+	public static int compare(Field f1, Field f2) {
+		return compare(f1.buffer, f1.offset, f1.length, f2.buffer, f2.offset, f2.length);
+	}
+	// copy bytes from one offset to another within the field
+	public Field copy(int fromOffset, int toOffset, int length) {
+		System.arraycopy(buffer, offset + fromOffset, buffer, offset + toOffset, length);
+		return this;
+	}
+	public Field dec(int n) {
+		offset -= n;
+		return this;
+	}
+	public byte[] get() {
+		byte[] result= new byte[length];
+		System.arraycopy(buffer, offset, result, 0, length);
+		return result;
+	}
+	public byte[] get(int offset, int length) {
+		byte[] result= new byte[length];
+		System.arraycopy(buffer, this.offset + offset, result, 0, length);
+		return result;
+	}
+	public Field getField(int offset, int length) {
+		return new Field(buffer, this.offset + offset, length);
+	}
+	public int getInt1() {
+		return buffer[this.offset];
+	}
+	public int getInt1(int offset) {
+		return buffer[this.offset + offset];
+	}
+	public int getInt2() {
+		int i= this.offset;
+		int v= buffer[i++];
+		v= (v << 8) | (buffer[i++] & 255);
+		return v;
+	}
+	public int getInt2(int offset) {
+		int i= this.offset + offset;
+		int v= buffer[i++];
+		v= (v << 8) | (buffer[i++] & 255);
+		return v;
+	}
+	public int getInt3() {
+		int i= this.offset;
+		int v= buffer[i++];
+		v= (v << 8) | (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		return v;
+	}
+	public int getInt3(int offset) {
+		int i= this.offset + offset;
+		int v= buffer[i++];
+		v= (v << 8) | (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		return v;
+	}
+	public int getInt4() {
+		int i= this.offset;
+		int v= buffer[i++];
+		v= (v << 8) | (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		return v;
+	}
+	public int getInt4(int offset) {
+		int i= this.offset + offset;
+		int v= buffer[i++];
+		v= (v << 8) | (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		return v;
+	}
+	public int getUInt1() {
+		return buffer[this.offset] & 255;
+	}
+	public int getUInt1(int offset) {
+		return buffer[this.offset + offset] & 255;
+	}
+	public int getUInt2() {
+		int i= this.offset;
+		int v= (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		return v;
+	}
+	public int getUInt2(int offset) {
+		int i= this.offset + offset;
+		int v= (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		return v;
+	}
+	public int getUInt3() {
+		int i= this.offset;
+		int v= (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		return v;
+	}
+	public int getUInt3(int offset) {
+		int i= this.offset + offset;
+		int v= (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		v= (v << 8) | (buffer[i++] & 255);
+		return v;
+	}
+	public char[] getUTF(int offset) throws UTFDataFormatException {
+		int pos= this.offset + offset;
+		int utflen= getUInt2(pos);
+		pos += 2;
+		char str[]= new char[utflen];
+		int count= 0;
+		int strlen= 0;
+		while (count < utflen) {
+			int c= buffer[pos++] & 0xFF;
+			int char2, char3;
+			switch (c >> 4) {
+				case 0 :
+				case 1 :
+				case 2 :
+				case 3 :
+				case 4 :
+				case 5 :
+				case 6 :
+				case 7 :
+					// 0xxxxxxx
+					count++;
+					str[strlen++]= (char) c;
+					break;
+				case 12 :
+				case 13 :
+					// 110x xxxx   10xx xxxx
+					count += 2;
+					if (count > utflen)
+						throw new UTFDataFormatException();
+					char2= buffer[pos++] & 0xFF;
+					if ((char2 & 0xC0) != 0x80)
+						throw new UTFDataFormatException();
+					str[strlen++]= (char) (((c & 0x1F) << 6) | (char2 & 0x3F));
+					break;
+				case 14 :
+					// 1110 xxxx  10xx xxxx  10xx xxxx
+					count += 3;
+					if (count > utflen)
+						throw new UTFDataFormatException();
+					char2= buffer[pos++] & 0xFF;
+					char3= buffer[pos++] & 0xFF;
+					if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80))
+						throw new UTFDataFormatException();
+					str[strlen++]= (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | ((char3 & 0x3F) << 0));
+					break;
+				default :
+					// 10xx xxxx,  1111 xxxx
+					throw new UTFDataFormatException();
+			}
+		}
+		if (strlen < utflen)
+			System.arraycopy(str, 0, str= new char[strlen], 0, strlen);
+		return str;
+	}
+	public Field inc(int n) {
+		offset += n;
+		return this;
+	}
+	public int length() {
+		return length;
+	}
+	public Field length(int length) {
+		this.length= length;
+		return this;
+	}
+	/**
+	Returns the offset into the underlying byte array that this field is defined over.
+	*/
+	public int offset() {
+		return offset;
+	}
+	public Field offset(int offset) {
+		this.offset= offset;
+		return this;
+	}
+	public Field pointTo(int offset) {
+		return new Field(buffer, this.offset + offset, 0);
+	}
+	public Field put(byte[] b) {
+		return put(0, b);
+	}
+	public Field put(int offset, byte[] b) {
+		System.arraycopy(b, 0, buffer, this.offset + offset, b.length);
+		return this;
+	}
+	public Field put(int offset, Field f) {
+		System.arraycopy(f.buffer, f.offset, buffer, this.offset + offset, f.length);
+		return this;
+	}
+	public Field put(Field f) {
+		System.arraycopy(f.buffer, f.offset, buffer, offset, f.length);
+		return this;
+	}
+	public Field putInt1(int n) {
+		buffer[offset]= (byte) (n);
+		return this;
+	}
+	public Field putInt1(int offset, int n) {
+		buffer[this.offset + offset]= (byte) (n);
+		return this;
+	}
+	public Field putInt2(int n) {
+		int i= offset;
+		buffer[i++]= (byte) (n >> 8);
+		buffer[i++]= (byte) (n >> 0);
+		return this;
+	}
+	public Field putInt2(int offset, int n) {
+		int i= this.offset + offset;
+		buffer[i++]= (byte) (n >> 8);
+		buffer[i++]= (byte) (n >> 0);
+		return this;
+	}
+	public Field putInt3(int n) {
+		int i= offset;
+		buffer[i++]= (byte) (n >> 16);
+		buffer[i++]= (byte) (n >> 8);
+		buffer[i++]= (byte) (n >> 0);
+		return this;
+	}
+	public Field putInt3(int offset, int n) {
+		int i= this.offset + offset;
+		buffer[i++]= (byte) (n >> 16);
+		buffer[i++]= (byte) (n >> 8);
+		buffer[i++]= (byte) (n >> 0);
+		return this;
+	}
+	public Field putInt4(int n) {
+		int i= offset;
+		buffer[i++]= (byte) (n >> 24);
+		buffer[i++]= (byte) (n >> 16);
+		buffer[i++]= (byte) (n >> 8);
+		buffer[i++]= (byte) (n >> 0);
+		return this;
+	}
+	public Field putInt4(int offset, int n) {
+		int i= this.offset + offset;
+		buffer[i++]= (byte) (n >> 24);
+		buffer[i++]= (byte) (n >> 16);
+		buffer[i++]= (byte) (n >> 8);
+		buffer[i++]= (byte) (n >> 0);
+		return this;
+	}
+	public int putUTF(int offset, char[] str) {
+		int strlen= str.length;
+		int utflen= 0;
+		for (int i= 0; i < strlen; i++) {
+			int c= str[i];
+			if ((c >= 0x0001) && (c <= 0x007F)) {
+				utflen++;
+			} else if (c > 0x07FF) {
+				utflen += 3;
+			} else {
+				utflen += 2;
+			}
+		}
+		if (utflen > 65535)
+			throw new IllegalArgumentException();
+		int pos= this.offset + offset;
+		buffer[pos++]= (byte) ((utflen >>> 8) & 0xFF);
+		buffer[pos++]= (byte) ((utflen >>> 0) & 0xFF);
+		for (int i= 0; i < strlen; i++) {
+			int c= str[i];
+			if ((c >= 0x0001) && (c <= 0x007F)) {
+				buffer[pos++]= ((byte) c);
+			} else if (c > 0x07FF) {
+				buffer[pos++]= ((byte) (0xE0 | ((c >> 12) & 0x0F)));
+				buffer[pos++]= ((byte) (0x80 | ((c >> 6) & 0x3F)));
+				buffer[pos++]= ((byte) (0x80 | ((c >> 0) & 0x3F)));
+			} else {
+				buffer[pos++]= ((byte) (0xC0 | ((c >> 6) & 0x1F)));
+				buffer[pos++]= ((byte) (0x80 | ((c >> 0) & 0x3F)));
+			}
+		}
+		return 2 + utflen;
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/FileListBlock.java

@@ -0,0 +1,110 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+ package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+public class FileListBlock extends Block {
+
+	protected int offset= 0;
+	protected String prevPath= null;
+	protected String[] paths= null;
+
+	public FileListBlock(int blockSize) {
+		super(blockSize);
+	}
+	/**
+	 * add the name of the indexedfile to the buffr of the field. 
+	 * The name is not the entire name of the indexedfile, but the 
+	 * difference between its name and the name of the previous indexedfile ...	
+	 */
+	public boolean addFile(IndexedFile indexedFile) {
+		int offset= this.offset;
+		if (isEmpty()) {
+			field.putInt4(offset, indexedFile.getFileNumber());
+			offset += 4;
+		}
+		String path= indexedFile.getPath();
+		int prefixLen= prevPath == null ? 0 : Util.prefixLength(prevPath, path);
+		int sizeEstimate= 2 + 2 + (path.length() - prefixLen) * 3;
+		if (offset + sizeEstimate > blockSize - 2)
+			return false;
+		field.putInt2(offset, prefixLen);
+		offset += 2;
+		char[] chars= new char[path.length() - prefixLen];
+		path.getChars(prefixLen, path.length(), chars, 0);
+		offset += field.putUTF(offset, chars);
+		this.offset= offset;
+		prevPath= path;
+		return true;
+	}
+	public void clear() {
+		reset();
+		super.clear();
+	}
+	public void flush() {
+		if (offset > 0) {
+			field.putInt2(offset, 0);
+			field.putInt2(offset + 2, 0);
+			offset= 0;
+		}
+	}
+	public IndexedFile getFile(int fileNum) throws IOException {
+		IndexedFile resp= null;
+		try {
+			String[] paths= getPaths();
+			int i= fileNum - field.getInt4(0);
+			resp= new IndexedFile(paths[i], fileNum);
+		} catch (Exception e) {
+			//fileNum too big
+		}
+		return resp;
+	}
+	/**
+	 * Creates a vector of paths reading the buffer of the field.
+	 */
+	protected String[] getPaths() throws IOException {
+		if (paths == null) {
+			ArrayList v= new ArrayList();
+			int offset= 4;
+			char[] prevPath= null;
+			for (;;) {
+				int prefixLen= field.getUInt2(offset);
+				offset += 2;
+				int utfLen= field.getUInt2(offset);
+				char[] path= field.getUTF(offset);
+				offset += 2 + utfLen;
+				if (prefixLen != 0) {
+					char[] temp= new char[prefixLen + path.length];
+					System.arraycopy(prevPath, 0, temp, 0, prefixLen);
+					System.arraycopy(path, 0, temp, prefixLen, path.length);
+					path= temp;
+				}
+				if (path.length == 0)
+					break;
+				v.add(new String(path));
+				prevPath= path;
+			}
+			paths= new String[v.size()];
+			v.toArray(paths);
+		}
+		return paths;
+	}
+	public boolean isEmpty() {
+		return offset == 0;
+	}
+	public void reset() {
+		offset= 0;
+		prevPath= null;
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/GammaCompressedIndexBlock.java

@@ -0,0 +1,118 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.UTFDataFormatException;
+
+/**
+ * Uses prefix coding on words, and gamma coding of document numbers differences.
+ */
+public class GammaCompressedIndexBlock extends IndexBlock {
+	CodeByteStream writeCodeStream= new CodeByteStream();
+	CodeByteStream readCodeStream;
+	char[] prevWord= null;
+	int offset= 0;
+
+	public GammaCompressedIndexBlock(int blockSize) {
+		super(blockSize);
+		readCodeStream= new CodeByteStream(field.buffer());
+	}
+	/**
+	 * @see IndexBlock#addEntry
+	 */
+	public boolean addEntry(WordEntry entry) {
+		writeCodeStream.reset();
+		encodeEntry(entry, prevWord, writeCodeStream);
+		if (offset + writeCodeStream.byteLength() > this.blockSize - 2) {
+			return false;
+		}
+		byte[] bytes= writeCodeStream.toByteArray();
+		field.put(offset, bytes);
+		offset += bytes.length;
+		prevWord= entry.getWord();
+		return true;
+	}
+	protected void encodeEntry(WordEntry entry, char[] prevWord, CodeByteStream codeStream) {
+		char[] word= entry.getWord();
+		int prefixLen= prevWord == null ? 0 : Util.prefixLength(prevWord, word);
+		codeStream.writeByte(prefixLen);
+		codeStream.writeUTF(word, prefixLen, word.length);
+		int n= entry.getNumRefs();
+		codeStream.writeGamma(n);
+		int prevRef= 0;
+		for (int i= 0; i < n; ++i) {
+			int ref= entry.getRef(i);
+			if (ref <= prevRef)
+				throw new IllegalArgumentException();
+			codeStream.writeGamma(ref - prevRef);
+			prevRef= ref;
+		}
+	}
+	/**
+	 * @see IndexBlock#flush
+	 */
+	public void flush() {
+		if (offset > 0) {
+			field.putInt2(offset, 0);
+			offset= 0;
+			prevWord= null;
+		}
+	}
+	/**
+	 * @see IndexBlock#isEmpty
+	 */
+	public boolean isEmpty() {
+		return offset == 0;
+	}
+	/**
+	 * @see IndexBlock#nextEntry
+	 */
+	public boolean nextEntry(WordEntry entry) {
+		try {
+			readCodeStream.reset(field.buffer(), offset);
+			int prefixLength= readCodeStream.readByte();
+			char[] word= readCodeStream.readUTF();
+			if (prevWord != null && prefixLength > 0) {
+				char[] temp= new char[prefixLength + word.length];
+				System.arraycopy(prevWord, 0, temp, 0, prefixLength);
+				System.arraycopy(word, 0, temp, prefixLength, word.length);
+				word= temp;
+			}
+			if (word.length == 0) {
+				return false;
+			}
+			entry.reset(word);
+			int n= readCodeStream.readGamma();
+			int prevRef= 0;
+			for (int i= 0; i < n; ++i) {
+				int ref= prevRef + readCodeStream.readGamma();
+				if (ref < prevRef)
+					throw new InternalError();
+				entry.addRef(ref);
+				prevRef= ref;
+			}
+			offset= readCodeStream.byteLength();
+			prevWord= word;
+			return true;
+		} catch (UTFDataFormatException e) {
+			return false;
+		}
+	}
+	/**
+	 * @see IndexBlock#reset
+	 */
+	public void reset() {
+		super.reset();
+		offset= 0;
+		prevWord= null;
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/IFileDocument.java

@@ -0,0 +1,98 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.IOException;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.cdt.internal.core.search.CharOperation;
+
+
+/**
+ * An <code>IFileDocument</code> represents an IFile.
+ */
+
+public class IFileDocument extends PropertyDocument {
+	protected IFile file;
+
+	// cached contents if needed - only one of them is used at a time
+	protected char[] charContents;
+	protected byte[] byteContents;
+	/**
+	 * IFileDocument constructor comment.
+	 */
+	public IFileDocument(IFile file) {
+		this(file, (char[])null);
+	}
+	/**
+	 * IFileDocument constructor comment.
+	 */
+	public IFileDocument(IFile file, byte[] byteContents) {
+		this.file= file;
+		this.byteContents= byteContents;
+	}
+	/**
+	 * IFileDocument constructor comment.
+	 */
+	public IFileDocument(IFile file, char[] charContents) {
+		this.file= file;
+		this.charContents= charContents;
+	}
+	/**
+	 * @see org.eclipse.jdt.internal.core.index.IDocument#getByteContent()
+	 */
+	public byte[] getByteContent() throws IOException {
+		if (byteContents != null) return byteContents;
+		IPath location = file.getLocation();
+		if (location == null) return new byte[0];
+		return byteContents = org.eclipse.cdt.internal.core.search.Util.getFileByteContent(location.toFile());
+	}
+	/**
+	 * @see org.eclipse.jdt.internal.core.index.IDocument#getCharContent()
+	 */
+	public char[] getCharContent() throws IOException {
+		if (charContents != null) return charContents;
+		IPath location = file.getLocation();
+		if (location == null) return CharOperation.NO_CHAR;
+		return charContents = org.eclipse.cdt.internal.core.search.Util.getFileCharContent(
+					location.toFile(), 
+					getEncoding());
+	}
+	/**
+	 * @see org.eclipse.jdt.internal.core.index.IDocument#getEncoding()
+	 */
+	public String getEncoding() {
+		//TODO: Indexer - get encoding
+	    return null;
+	}
+	/**
+	 * @see org.eclipse.jdt.internal.core.index.IDocument#getName()
+	 */
+	public String getName() {
+		return file.getFullPath().toString();
+	}
+	/**
+	 * @see org.eclipse.jdt.internal.core.index.IDocument#getStringContent()
+	 */
+	public String getStringContent() throws java.io.IOException {
+		return new String(getCharContent());
+	}
+	/**
+	 * @see org.eclipse.jdt.internal.core.index.IDocument#getType()
+	 */
+	public String getType() {
+		String extension= file.getFileExtension();
+		if (extension == null)
+			return ""; //$NON-NLS-1$
+		return extension;
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/IIndexConstants.java

@@ -0,0 +1,29 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+/**
+ * This interface provides constants used by the search engine.
+ */
+public interface IIndexConstants {
+	/**
+	 * The signature of the index file.
+	 */
+	public static final String SIGNATURE= "INDEX FILE 0.012"; //$NON-NLS-1$
+	/**
+	 * The separator for files in the index file.
+	 */
+	public static final char FILE_SEPARATOR= '/';
+	/**
+	 * The size of a block for a <code>Block</code>.
+	 */
+	public static final int BLOCK_SIZE= 8192;
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/InMemoryIndex.java

@@ -0,0 +1,197 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.eclipse.cdt.internal.core.index.IDocument;
+
+/**
+ * This index stores the document names in an <code>ObjectVector</code>, and the words in
+ * an <code>HashtableOfObjects</code>.
+ */
+
+public class InMemoryIndex {
+
+	/**
+	 * hashtable of WordEntrys = words+numbers of the files they appear in.
+	 */
+	protected WordEntryHashedArray words;
+	/**
+	 * List of IndexedFiles = file name + a unique number.
+	 */
+	protected IndexedFileHashedArray files;
+	/**
+	 * Size of the index.
+	 */
+	protected long footprint;
+
+	private WordEntry[] sortedWordEntries;
+	private IndexedFile[] sortedFiles;
+	public InMemoryIndex() {
+		init();
+	}
+
+	public IndexedFile addDocument(IDocument document) {
+		IndexedFile indexedFile= this.files.add(document);
+		this.footprint += indexedFile.footprint() + 4;
+		this.sortedFiles = null;
+		return indexedFile;
+	}
+	/**
+	 * Adds the references of the word to the index (reference = number of the file the word belongs to).
+	 */
+	protected void addRef(char[] word, int[] references) {
+		int size= references.length;
+		int i= 0;
+		while (i < size) {
+			if (references[i] != 0)
+				addRef(word, references[i]);
+			i++;
+		}
+	}
+	/**
+	 * Looks if the word already exists in the index and add the fileNum to this word.
+	 * If the word does not exist, it adds it in the index.
+	 */
+	protected void addRef(char[] word, int fileNum) {
+		WordEntry entry= (WordEntry) this.words.get(word);
+		if (entry == null) {
+			entry= new WordEntry(word);
+			entry.addRef(fileNum);
+			this.words.add(entry);
+			this.sortedWordEntries= null;
+			this.footprint += entry.footprint();
+		} else {
+			this.footprint += entry.addRef(fileNum);
+		}
+	}
+
+	public void addRef(IndexedFile indexedFile, char[] word) {
+		addRef(word, indexedFile.getFileNumber());
+	}
+
+	public void addRef(IndexedFile indexedFile, String word) {
+		addRef(word.toCharArray(), indexedFile.getFileNumber());
+	}
+	/**
+	 * Returns the footprint of the index.
+	 */
+	public long getFootprint() {
+		return this.footprint;
+	}
+	/**
+	 * Returns the indexed file with the given path, or null if such file does not exist.
+	 */
+	public IndexedFile getIndexedFile(String path) {
+		return files.get(path);
+	}
+	/**
+	 * @see IIndex#getNumDocuments()
+	 */
+	public int getNumFiles() {
+		return files.size();
+	}
+	/**
+	 * @see IIndex#getNumWords()
+	 */
+	public int getNumWords() {
+		return words.elementSize;
+	}
+	/**
+	 * Returns the words contained in the hashtable of words, sorted by alphabetical order.
+	 */
+	protected IndexedFile[] getSortedFiles() {
+		if (this.sortedFiles == null) {
+			IndexedFile[] indexedFiles= files.asArray();
+			Util.sort(indexedFiles);
+			this.sortedFiles= indexedFiles;
+		}
+		return this.sortedFiles;
+	}
+	/**
+	 * Returns the word entries contained in the hashtable of words, sorted by alphabetical order.
+	 */
+	protected WordEntry[] getSortedWordEntries() {
+		if (this.sortedWordEntries == null) {
+			WordEntry[] words= this.words.asArray();
+			Util.sort(words);
+			this.sortedWordEntries= words;
+		}
+		return this.sortedWordEntries;
+	}
+	/**
+	 * Returns the word entry corresponding to the given word.
+	 */
+	protected WordEntry getWordEntry(char[] word) {
+		return (WordEntry) words.get(word);
+	}
+	/**
+	 * Initialises the fields of the index
+	 */
+	public void init() {
+		words= new WordEntryHashedArray(501);
+		files= new IndexedFileHashedArray(101);
+		footprint= 0;
+		sortedWordEntries= null;
+		sortedFiles= null;
+	}
+	/**
+	 * Saves the index in the given file.
+	 * Structure of the saved Index :
+	 *   - IndexedFiles in sorted order.
+	 *		+ example: 
+	 *			"c:/com/a.cpp 1"
+	 *			"c:/com/b.cpp 2"
+	 *   - References with the words in sorted order
+	 *		+ example: 
+	 *			"classDecl/a 1"
+	 *			"classDecl/b 2"
+	 *			"ref/String 1 2"
+	 */
+	public void save(File file) throws IOException {
+		BlocksIndexOutput output= new BlocksIndexOutput(file);
+		save(output);
+	}
+	/**
+	 * Saves the index in the given IndexOutput.
+	 * Structure of the saved Index :
+	 *   - IndexedFiles in sorted order.
+	 *		+ example: 
+	 *			"c:/com/a.cpp 1"
+	 *			"c:/com/b.cpp 2"
+	 *   - References with the words in sorted order
+	 *		+ example: 
+	 *			"classDecl/a 1"
+	 *			"classDecl/b 2"
+	 *			"ref/String 1 2"
+	 */
+	protected void save(IndexOutput output) throws IOException {
+		boolean ok= false;
+		try {
+			output.open();
+			IndexedFile[] indexedFiles= files.asArray();
+			for (int i= 0, length = indexedFiles.length; i < length; ++i)
+				output.addFile(indexedFiles[i]); // written out in order BUT not alphabetical
+			getSortedWordEntries(); // init the slot
+			for (int i= 0, numWords= sortedWordEntries.length; i < numWords; ++i)
+				output.addWord(sortedWordEntries[i]);
+			output.flush();
+			output.close();
+			ok= true;
+		} finally {
+			if (!ok && output != null)
+				output.close();
+		}
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/Index.java

@@ -0,0 +1,364 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.eclipse.cdt.internal.core.index.IDocument;
+import org.eclipse.cdt.internal.core.index.IEntryResult;
+import org.eclipse.cdt.internal.core.index.IIndex;
+import org.eclipse.cdt.internal.core.index.IIndexer;
+import org.eclipse.cdt.internal.core.index.IQueryResult;
+
+/**
+ * An Index is used to create an index on the disk, and to make queries. It uses a set of 
+ * indexers and a mergeFactory. The index fills an inMemoryIndex up 
+ * to it reaches a certain size, and then merges it with a main index on the disk.
+ * <br> <br>
+ * The changes are only taken into account by the queries after a merge.
+ */
+
+public class Index implements IIndex {
+	/**
+	 * Maximum size of the index in memory.
+	 */
+	public static final int MAX_FOOTPRINT= 10000000;
+
+	/**
+	 * Index in memory, who is merged with mainIndex each times it 
+	 * reaches a certain size.
+	 */
+	protected InMemoryIndex addsIndex;
+	protected IndexInput addsIndexInput;
+
+	/**
+	 * State of the indexGenerator: addsIndex empty <=> MERGED, or
+	 * addsIndex not empty <=> CAN_MERGE
+	 */
+	protected int state;
+
+	/**
+	 * Files removed form the addsIndex.
+	 */
+	protected Map removedInAdds;
+
+	/**
+	 * Files removed form the oldIndex.
+	 */
+	protected Map removedInOld;
+	protected static final int CAN_MERGE= 0;
+	protected static final int MERGED= 1;
+	private File indexFile;
+
+	/**
+	 * String representation of this index.
+	 */
+	public String toString;
+	
+	public Index(File indexDirectory, boolean reuseExistingFile) throws IOException {
+		this(indexDirectory,".index", reuseExistingFile); //$NON-NLS-1$
+	}
+	
+	public Index(File indexDirectory, String indexName, boolean reuseExistingFile) throws IOException {
+		super();
+		state= MERGED;
+		indexFile= new File(indexDirectory, indexName);
+		initialize(reuseExistingFile);
+	}
+	
+	public Index(String indexName, boolean reuseExistingFile) throws IOException {
+		this(indexName, null, reuseExistingFile);
+	}
+
+	public Index(String indexName, String toString, boolean reuseExistingFile) throws IOException {
+		super();
+		state= MERGED;
+		indexFile= new File(indexName);
+		this.toString = toString;
+		initialize(reuseExistingFile);
+	}
+	/**
+	 * Indexes the given document, using the appropriate indexer registered in the indexerRegistry.
+	 * If the document already exists in the index, it overrides the previous one. The changes will be 
+	 * taken into account after a merge.
+	 */
+	public void add(IDocument document, IIndexer indexer) throws IOException {
+		if (timeToMerge()) {
+			merge();
+		}
+		IndexedFile indexedFile= addsIndex.getIndexedFile(document.getName());
+		if (indexedFile != null /*&& removedInAdds.get(document.getName()) == null*/
+			)
+			remove(indexedFile, MergeFactory.ADDS_INDEX);
+		IndexerOutput output= new IndexerOutput(addsIndex);
+		indexer.index(document, output);
+		state= CAN_MERGE;
+	}
+	/**
+	 * Returns true if the index in memory is not empty, so 
+	 * merge() can be called to fill the mainIndex with the files and words
+	 * contained in the addsIndex. 
+	 */
+	protected boolean canMerge() {
+		return state == CAN_MERGE;
+	}
+	/**
+	 * Initialises the indexGenerator.
+	 */
+	public void empty() throws IOException {
+
+		if (indexFile.exists()){
+			indexFile.delete();
+			//initialisation of mainIndex
+			InMemoryIndex mainIndex= new InMemoryIndex();
+			IndexOutput mainIndexOutput= new BlocksIndexOutput(indexFile);
+			if (!indexFile.exists())
+				mainIndex.save(mainIndexOutput);
+		}
+
+		//initialisation of addsIndex
+		addsIndex= new InMemoryIndex();
+		addsIndexInput= new SimpleIndexInput(addsIndex);
+
+		//vectors who keep track of the removed Files
+		removedInAdds= new HashMap(11);
+		removedInOld= new HashMap(11);
+	}
+	/**
+	 * @see IIndex#getIndexFile
+	 */
+	public File getIndexFile() {
+		return indexFile;
+	}
+	/**
+	 * @see IIndex#getNumDocuments
+	 */
+	public int getNumDocuments() throws IOException {
+		//save();
+		IndexInput input= new BlocksIndexInput(indexFile);
+		try {
+			input.open();
+			return input.getNumFiles();
+		} finally {
+			input.close();
+		}		
+	}
+	/**
+	 * @see IIndex#getNumWords
+	 */
+	public int getNumWords() throws IOException {
+		//save();
+		IndexInput input= new BlocksIndexInput(indexFile);
+		try {
+			input.open();
+			return input.getNumWords();
+		} finally {
+			input.close();
+		}		
+	}
+	/**
+	 * Returns the path corresponding to a given document number
+	 */
+	public String getPath(int documentNumber) throws IOException {
+		//save();
+		IndexInput input= new BlocksIndexInput(indexFile);
+		try {
+			input.open();
+			IndexedFile file = input.getIndexedFile(documentNumber);
+			if (file == null) return null;
+			return file.getPath();
+		} finally {
+			input.close();
+		}		
+	}
+	/**
+	 * see IIndex.hasChanged
+	 */
+	public boolean hasChanged() {
+		return canMerge();
+	}
+	/**
+	 * Initialises the indexGenerator.
+	 */
+	public void initialize(boolean reuseExistingFile) throws IOException {
+		//initialisation of addsIndex
+		addsIndex= new InMemoryIndex();
+		addsIndexInput= new SimpleIndexInput(addsIndex);
+
+		//vectors who keep track of the removed Files
+		removedInAdds= new HashMap(11);
+		removedInOld= new HashMap(11);
+
+		// check whether existing index file can be read
+		if (reuseExistingFile && indexFile.exists()) {
+			IndexInput mainIndexInput= new BlocksIndexInput(indexFile);
+			try {
+				mainIndexInput.open();
+			} catch(IOException e) {
+				BlocksIndexInput input = (BlocksIndexInput)mainIndexInput;
+				try {
+					input.opened = true;
+					input.close();
+				} finally {
+					input.opened = false;
+				}
+				indexFile.delete();
+				mainIndexInput = null;
+				throw e;
+			}
+			mainIndexInput.close();
+		} else {
+			InMemoryIndex mainIndex= new InMemoryIndex();			
+			IndexOutput mainIndexOutput= new BlocksIndexOutput(indexFile);
+			mainIndex.save(mainIndexOutput);
+		}
+	}
+	/**
+	 * Merges the in memory index and the index on the disk, and saves the results on the disk.
+	 */
+	protected void merge() throws IOException {
+		//initialisation of tempIndex
+		File tempFile= new File(indexFile.getAbsolutePath() + "TempVA"); //$NON-NLS-1$
+
+		IndexInput mainIndexInput= new BlocksIndexInput(indexFile);
+		BlocksIndexOutput tempIndexOutput= new BlocksIndexOutput(tempFile);
+
+		try {
+			//invoke a mergeFactory
+			new MergeFactory(
+				mainIndexInput, 
+				addsIndexInput, 
+				tempIndexOutput, 
+				removedInOld, 
+				removedInAdds).merge();
+			
+			//rename the file created to become the main index
+			File mainIndexFile= (File) mainIndexInput.getSource();
+			File tempIndexFile= (File) tempIndexOutput.getDestination();
+			mainIndexFile.delete();
+			tempIndexFile.renameTo(mainIndexFile);
+		} finally {		
+			//initialise remove vectors and addsindex, and change the state
+			removedInAdds.clear();
+			removedInOld.clear();
+			addsIndex.init();
+			addsIndexInput= new SimpleIndexInput(addsIndex);
+			state= MERGED;
+		}
+	}
+	/**
+	 * @see IIndex#query
+	 */
+	public IQueryResult[] query(String word) throws IOException {
+		//save();
+		IndexInput input= new BlocksIndexInput(indexFile);
+		try {
+			return input.query(word);
+		} finally {
+			input.close();
+		}
+	}
+	public IEntryResult[] queryEntries(char[] prefix) throws IOException {
+		//save();
+		IndexInput input= new BlocksIndexInput(indexFile);
+		try {
+			return input.queryEntriesPrefixedBy(prefix);
+		} finally {
+			input.close();
+		}
+	}
+	/**
+	 * @see IIndex#queryInDocumentNames
+	 */
+	public IQueryResult[] queryInDocumentNames(String word) throws IOException {
+		//save();
+		IndexInput input= new BlocksIndexInput(indexFile);
+		try {
+			return input.queryInDocumentNames(word);
+		} finally {
+			input.close();
+		}
+	}
+	/**
+	 * @see IIndex#queryPrefix
+	 */
+	public IQueryResult[] queryPrefix(char[] prefix) throws IOException {
+		//save();
+		IndexInput input= new BlocksIndexInput(indexFile);
+		try {
+			return input.queryFilesReferringToPrefix(prefix);
+		} finally {
+			input.close();
+		}
+	}
+	/**
+	 * @see IIndex#remove
+	 */
+	public void remove(String documentName) throws IOException {
+		IndexedFile file= addsIndex.getIndexedFile(documentName);
+		if (file != null) {
+			//the file is in the adds Index, we remove it from this one
+			Int lastRemoved= (Int) removedInAdds.get(documentName);
+			if (lastRemoved != null) {
+				int fileNum= file.getFileNumber();
+				if (lastRemoved.value < fileNum)
+					lastRemoved.value= fileNum;
+			} else
+				removedInAdds.put(documentName, new Int(file.getFileNumber()));
+		} else {
+			//we remove the file from the old index
+			removedInOld.put(documentName, new Int(1));
+		}
+		state= CAN_MERGE;
+	}
+	/**
+	 * Removes the given document from the given index (MergeFactory.ADDS_INDEX for the
+	 * in memory index, MergeFactory.OLD_INDEX for the index on the disk).
+	 */
+	protected void remove(IndexedFile file, int index) throws IOException {
+		String name= file.getPath();
+		if (index == MergeFactory.ADDS_INDEX) {
+			Int lastRemoved= (Int) removedInAdds.get(name);
+			if (lastRemoved != null) {
+				if (lastRemoved.value < file.getFileNumber())
+					lastRemoved.value= file.getFileNumber();
+			} else
+				removedInAdds.put(name, new Int(file.getFileNumber()));
+		} else if (index == MergeFactory.OLD_INDEX)
+			removedInOld.put(name, new Int(1));
+		else
+			throw new Error();
+		state= CAN_MERGE;
+	}
+	/**
+	 * @see IIndex#save
+	 */
+	public void save() throws IOException {
+		if (canMerge())
+			merge();
+	}
+	/**
+	 * Returns true if the in memory index reaches a critical size, 
+	 * to merge it with the index on the disk.
+	 */
+	protected boolean timeToMerge() {
+		return (addsIndex.getFootprint() >= MAX_FOOTPRINT);
+	}
+	public String toString() {
+	String str = this.toString;
+	if (str == null) str = super.toString();
+	str += "(length: "+ getIndexFile().length() +")"; //$NON-NLS-1$ //$NON-NLS-2$
+	return str;
+}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/IndexBlock.java

@@ -0,0 +1,74 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import org.eclipse.cdt.internal.core.search.CharOperation;
+
+/**
+ * An indexBlock stores wordEntries.
+ */
+public abstract class IndexBlock extends Block {
+
+	public IndexBlock(int blockSize) {
+		super(blockSize);
+	}
+	/**
+	 * Adds the given wordEntry to the indexBlock.
+	 */
+	public abstract boolean addEntry(WordEntry entry);
+	/**
+	 * @see Block#clear()
+	 */
+	public void clear() {
+		reset();
+		super.clear();
+	}
+	public WordEntry findEntryMatching(char[] pattern, boolean isCaseSensitive) {
+		reset();
+		WordEntry entry= new WordEntry();
+		while (nextEntry(entry)) {
+			if (CharOperation.match(pattern, entry.getWord(), isCaseSensitive)) {
+				return entry;
+			}
+		}
+		return null;
+	}
+	public WordEntry findEntryPrefixedBy(char[] word, boolean isCaseSensitive) {
+		reset();
+		WordEntry entry= new WordEntry();
+		while (nextEntry(entry)) {
+			if (CharOperation.prefixEquals(entry.getWord(), word, isCaseSensitive)) {
+				return entry;
+			}
+		}
+		return null;
+	}
+	public WordEntry findExactEntry(char[] word) {
+		reset();
+		WordEntry entry= new WordEntry();
+		while (nextEntry(entry)) {
+			if (CharOperation.equals(entry.getWord(), word)) {
+				return entry;
+			}
+		}
+		return null;
+	}
+	/**
+	 * Returns whether the block is empty or not (if it doesn't contain any wordEntry).
+	 */
+	public abstract boolean isEmpty();
+	/**
+	 * Finds the next wordEntry and stores it in the given entry.
+	 */
+	public abstract boolean nextEntry(WordEntry entry);
+	public void reset() {
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/IndexInput.java

@@ -0,0 +1,131 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.IOException;
+import org.eclipse.cdt.internal.core.index.IDocument;
+import org.eclipse.cdt.internal.core.index.IEntryResult;
+import org.eclipse.cdt.internal.core.index.IQueryResult;
+
+
+/**
+ * This class provides an input on an index, after it has been generated.
+ * You can access all the files of an index via getNextFile(), getCurrentFile() 
+ * and moveToNextFile() (idem for the word entries). 
+ * The usage is the same for every subclass: creation (constructor), opening
+ * (the open() method), usage, and closing (the close() method), to release the
+ * data source used by this input.
+ */
+public abstract class IndexInput {
+	protected int filePosition;
+	protected WordEntry currentWordEntry;
+	protected int wordPosition;
+
+
+	public IndexInput() {
+		super();
+		wordPosition= 1;
+		filePosition= 1;
+	}
+	/**
+	 * clears the cache of this indexInput, if it keeps track of the information already read.
+	 */
+	public abstract void clearCache();
+	/**
+	 * Closes the IndexInput. For example, if the input is on a RandomAccessFile,
+	 * it calls the close() method of RandomAccessFile. 
+	 */
+	public abstract void close() throws IOException;
+	/**
+	 * Returns the current file the indexInput is pointing to in the index.
+	 */
+	public abstract IndexedFile getCurrentFile() throws IOException;
+	/**
+	 * Returns the current file the indexInput is pointing to in the index.
+	 */
+	public WordEntry getCurrentWordEntry() throws IOException {
+		if (!hasMoreWords())
+			return null;
+		return currentWordEntry;
+	}
+	/**
+	 * Returns the position of the current file the input is pointing to in the index.
+	 */
+	public int getFilePosition() {
+		return filePosition;
+	}
+	/**
+	 * Returns the indexedFile corresponding to the given document number in the index the input
+	 * reads in, or null if such indexedFile does not exist.
+	 */
+	public abstract IndexedFile getIndexedFile(int fileNum) throws IOException;
+	/**
+	 * Returns the indexedFile corresponding to the given document in the index the input
+	 * reads in (e.g. the indexedFile with the same path in this index), or null if such 
+	 * indexedFile does not exist.
+	 */
+	public abstract IndexedFile getIndexedFile(IDocument document) throws IOException;
+	/**
+	 * Returns the number of files in the index.
+	 */
+	public abstract int getNumFiles();
+	/**
+	 * Returns the number of unique words in the index.
+	 */
+	public abstract int getNumWords();
+	/**
+	 * Returns the Object the input is reading from. It can be an IIndex,
+	 * a File, ...
+	 */
+	public abstract Object getSource();
+	/**
+	 * Returns true if the input has not reached the end of the index for the files.
+	 */
+	public boolean hasMoreFiles() {
+		return getFilePosition() <= getNumFiles();
+	}
+	/**
+	 * Returns true if the input has not reached the end of the index for the files.
+	 */
+	public boolean hasMoreWords() {
+		return wordPosition <= getNumWords();
+	}
+	/**
+	 * Moves the pointer on the current file to the next file in the index.
+	 */
+	public abstract void moveToNextFile() throws IOException;
+	/**
+	 * Moves the pointer on the current word to the next file in the index.
+	 */
+	public abstract void moveToNextWordEntry() throws IOException;
+	/**
+	 * Open the Source where the input gets the information from.
+	 */
+	public abstract void open() throws IOException;
+	/**
+	 * Returns the list of the files containing the given word in the index.
+	 */
+	public abstract IQueryResult[] query(String word) throws IOException;
+	public abstract IEntryResult[] queryEntriesPrefixedBy(char[] prefix) throws IOException;
+	public abstract IQueryResult[] queryFilesReferringToPrefix(char[] prefix) throws IOException;
+	/**
+	 * Returns the list of the files whose name contain the given word in the index.
+	 */
+	public abstract IQueryResult[] queryInDocumentNames(String word) throws IOException;
+	/**
+	 * Set the pointer on the current file to the first file of the index.
+	 */
+	protected abstract void setFirstFile() throws IOException;
+	/**
+	 * Set the pointer on the current word to the first word of the index.
+	 */
+	protected abstract void setFirstWord() throws IOException;
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/IndexOutput.java

@@ -0,0 +1,43 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.IOException;
+
+/**
+ * An indexOutput is used to write an index into a different object (a File, ...). 
+ */
+public abstract class IndexOutput {
+	/**
+	 * Adds a File to the destination.
+	 */
+	public abstract void addFile(IndexedFile file) throws IOException;
+	/**
+	 * Adds a word to the destination.
+	 */
+	public abstract void addWord(WordEntry word) throws IOException;
+	/**
+	 * Closes the output, releasing the resources it was using.
+	 */
+	public abstract void close() throws IOException;
+	/**
+	 * Flushes the output.
+	 */
+	public abstract void flush() throws IOException;
+	/**
+	 * Returns the Object the output is writing to. It can be a file, another type of index, ... 
+	 */
+	public abstract Object getDestination();
+	/**
+	 * Opens the output, before writing any information.
+	 */
+	public abstract void open() throws IOException;
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/IndexSummary.java

@@ -0,0 +1,315 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.ArrayList;
+
+import org.eclipse.cdt.internal.core.search.CharOperation;
+
+/**
+ * An indexSummary is used when saving an index into a BlocksIndexOuput or 
+ * reading it from a BlocksIndexInput. It contains basic informations about 
+ * an index: first files/words in each block, number of files/words.
+ */
+
+public class IndexSummary {
+	/**
+	 * First file for each block.
+	 */
+	protected ArrayList firstFilesInBlocks= new ArrayList();
+	/**
+	 * First word for each block.
+	 */
+	protected ArrayList firstWordsInBlocks= new ArrayList();
+	/**
+	 * Number of files in the index.
+	 */
+	protected int numFiles;
+	/**
+	 * Number of words in the index.
+	 */
+	protected int numWords;
+
+	static class FirstFileInBlock {
+		IndexedFile indexedFile;
+		int blockNum;
+	}
+
+	static class FirstWordInBlock {
+		char[] word;
+		int blockNum;
+		public String toString(){
+			return "FirstWordInBlock: " + new String(word) + ", blockNum: " + blockNum; //$NON-NLS-1$ //$NON-NLS-2$
+		}
+	}
+
+	protected int firstWordBlockNum;
+	protected boolean firstWordAdded= true;
+	/**
+	 * Adds the given file as the first file for the given Block number. 
+	 */
+	public void addFirstFileInBlock(IndexedFile indexedFile, int blockNum) {
+		FirstFileInBlock entry= new FirstFileInBlock();
+		entry.indexedFile= indexedFile;
+		entry.blockNum= blockNum;
+		firstFilesInBlocks.add(entry);
+	}
+	/**
+	 * Adds the given word as the first word for the given Block number. 
+	 */
+	public void addFirstWordInBlock(char[] word, int blockNum) {
+		if (firstWordAdded) {
+			firstWordBlockNum= blockNum;
+			firstWordAdded= false;
+		}
+		FirstWordInBlock entry= new FirstWordInBlock();
+		entry.word= word;
+		entry.blockNum= blockNum;
+		firstWordsInBlocks.add(entry);
+	}
+	/**
+	 * Returns the numbers of all the blocks
+	 */
+	public int[] getAllBlockNums() {
+
+		int max = firstWordsInBlocks.size();
+		int[] blockNums = new int[max];
+		for (int i = 0; i < max; i++){
+			blockNums[i] = ((FirstWordInBlock)firstWordsInBlocks.get(i)).blockNum;
+		}
+		return blockNums;
+	}
+	public int getBlockNum(int blockLocation) {
+	return ((FirstWordInBlock) firstWordsInBlocks.get(blockLocation)).blockNum;
+	}
+	/**
+	 * Returns the number of the Block containing the file with the given number. 
+	 */
+	public int getBlockNumForFileNum(int fileNum) {
+		int min= 0;
+		int max= firstFilesInBlocks.size() - 1;
+		while (min <= max) {
+			int mid= (min + max) / 2;
+			FirstFileInBlock entry= (FirstFileInBlock) firstFilesInBlocks.get(mid);
+			int compare= fileNum - entry.indexedFile.getFileNumber();
+			if (compare == 0)
+				return entry.blockNum;
+			if (compare < 0)
+				max= mid - 1;
+			else
+				min= mid + 1;
+		}
+		if (max < 0)
+			return -1;
+		FirstFileInBlock entry= (FirstFileInBlock) firstFilesInBlocks.get(max);
+		return entry.blockNum;
+	}
+	/**
+	 * Returns the number of the Block containing the given word. 
+	 */
+	public int getBlockNumForWord(char[] word) {
+		int min= 0;
+		int max= firstWordsInBlocks.size() - 1;
+		while (min <= max) {
+			int mid= (min + max) / 2;
+			FirstWordInBlock entry= (FirstWordInBlock) firstWordsInBlocks.get(mid);
+			int compare= Util.compare(word, entry.word);
+			if (compare == 0)
+				return entry.blockNum;
+			if (compare < 0)
+				max= mid - 1;
+			else
+				min= mid + 1;
+		}
+		if (max < 0)
+			return -1;
+		FirstWordInBlock entry= (FirstWordInBlock) firstWordsInBlocks.get(max);
+		return entry.blockNum;
+	}
+	public int[] getBlockNumsForPrefix(char[] prefix) {
+		int min= 0;
+		int size= firstWordsInBlocks.size();
+		int max=  size - 1;
+		int match= -1;
+		while (min <= max && match < 0) {
+			int mid= (min + max) / 2;
+			FirstWordInBlock entry= (FirstWordInBlock) firstWordsInBlocks.get(mid);
+			int compare= CharOperation.compareWith(entry.word, prefix);
+			if (compare == 0) {
+				match= mid;
+				break;
+			}	
+			if (compare >= 0)
+				max= mid - 1;
+			else
+				min= mid + 1;
+		}
+		if (max < 0)
+			return new int[0];
+			
+		if (match < 0)
+			match= max;
+		
+		int firstBlock= match - 1;
+		// Look if previous blocks are affected
+		for (; firstBlock >= 0; firstBlock--) {
+			FirstWordInBlock entry= (FirstWordInBlock) firstWordsInBlocks.get(firstBlock);
+			if (!CharOperation.prefixEquals(prefix, entry.word))
+				break;
+		}
+		if (firstBlock < 0)
+			firstBlock= 0;	
+		
+		// Look if next blocks are affected
+		int firstNotIncludedBlock= match + 1;
+		for (; firstNotIncludedBlock < size; firstNotIncludedBlock++) {
+			FirstWordInBlock entry= (FirstWordInBlock) firstWordsInBlocks.get(firstNotIncludedBlock);
+			if (!CharOperation.prefixEquals(prefix, entry.word))
+				break;
+		}
+		
+		int numberOfBlocks= firstNotIncludedBlock - firstBlock;
+		int[] result= new int[numberOfBlocks];
+		int pos= firstBlock;
+		for (int i= 0; i < numberOfBlocks; i++, pos++) {
+			FirstWordInBlock entry= (FirstWordInBlock) firstWordsInBlocks.get(pos);
+			result[i]= entry.blockNum;
+		}
+		return result;
+	}
+	public int getFirstBlockLocationForPrefix(char[] prefix) {
+		int min = 0;
+		int size = firstWordsInBlocks.size();
+		int max = size - 1;
+		int match = -1;
+		while (min <= max) {
+			int mid = (min + max) / 2;
+			FirstWordInBlock entry = (FirstWordInBlock) firstWordsInBlocks.get(mid);
+			int compare = CharOperation.compareWith(entry.word, prefix);
+			if (compare == 0) {
+				match = mid;
+				break;
+			}
+			if (compare >= 0) {
+				max = mid - 1;
+			} else {
+				match = mid; // not perfect match, but could be inside
+				min = mid + 1;
+			}
+		}
+		if (max < 0) return -1;
+	
+		// no match at all, might be some matching entries inside max block
+		if (match < 0){
+			match = max;
+		} else {
+			// look for possible matches inside previous blocks
+			while (match > 0){
+				FirstWordInBlock entry = (FirstWordInBlock) firstWordsInBlocks.get(match);
+				if (!CharOperation.prefixEquals(prefix, entry.word))
+					break;
+				match--;
+			}
+		}
+		return match;
+	}
+	/**
+	 * Returns the number of the first IndexBlock (containing words).
+	 */
+	public int getFirstWordBlockNum() {
+		return firstWordBlockNum;
+	}
+	/** 
+	 * Blocks are contiguous, so the next one is a potential candidate if its first word starts with
+	 * the given prefix
+	 */
+	public int getNextBlockLocationForPrefix(char[] prefix, int blockLoc) {
+		if (++blockLoc < firstWordsInBlocks.size()){
+			FirstWordInBlock entry= (FirstWordInBlock) firstWordsInBlocks.get(blockLoc);
+			if (CharOperation.prefixEquals(prefix, entry.word)) return blockLoc;
+		}
+		return -1;
+	}
+	/**
+	 * Returns the number of files contained in the index.
+	 */
+	public int getNumFiles() {
+		return numFiles;
+	}
+	/**
+	 * Returns the number of words contained in the index.
+	 */
+	public int getNumWords() {
+		return numWords;
+	}
+	/**
+	 * Loads the summary in memory.
+	 */
+	public void read(RandomAccessFile raf) throws IOException {
+		numFiles= raf.readInt();
+		numWords= raf.readInt();
+		firstWordBlockNum= raf.readInt();
+		int numFirstFiles= raf.readInt();
+		for (int i= 0; i < numFirstFiles; ++i) {
+			FirstFileInBlock entry= new FirstFileInBlock();
+			String path= raf.readUTF();
+			int fileNum= raf.readInt();
+			entry.indexedFile= new IndexedFile(path, fileNum);
+			entry.blockNum= raf.readInt();
+			firstFilesInBlocks.add(entry);
+		}
+		int numFirstWords= raf.readInt();
+		for (int i= 0; i < numFirstWords; ++i) {
+			FirstWordInBlock entry= new FirstWordInBlock();
+			entry.word= raf.readUTF().toCharArray();
+			entry.blockNum= raf.readInt();
+			firstWordsInBlocks.add(entry);
+		}
+	}
+	/**
+	 * Sets the number of files of the index.
+	 */
+
+	public void setNumFiles(int numFiles) {
+		this.numFiles= numFiles;
+	}
+	/**
+	 * Sets the number of words of the index.
+	 */
+
+	public void setNumWords(int numWords) {
+		this.numWords= numWords;
+	}
+	/**
+	 * Saves the summary on the disk.
+	 */
+	public void write(RandomAccessFile raf) throws IOException {
+		raf.writeInt(numFiles);
+		raf.writeInt(numWords);
+		raf.writeInt(firstWordBlockNum);
+		raf.writeInt(firstFilesInBlocks.size());
+		for (int i= 0, size= firstFilesInBlocks.size(); i < size; ++i) {
+			FirstFileInBlock entry= (FirstFileInBlock) firstFilesInBlocks.get(i);
+			raf.writeUTF(entry.indexedFile.getPath());
+			raf.writeInt(entry.indexedFile.getFileNumber());
+			raf.writeInt(entry.blockNum);
+		}
+		raf.writeInt(firstWordsInBlocks.size());
+		for (int i= 0, size= firstWordsInBlocks.size(); i < size; ++i) {
+			FirstWordInBlock entry= (FirstWordInBlock) firstWordsInBlocks.get(i);
+			raf.writeUTF(new String(entry.word));
+			raf.writeInt(entry.blockNum);
+		}
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/IndexedFile.java

@@ -0,0 +1,65 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import org.eclipse.cdt.internal.core.index.IQueryResult;
+import org.eclipse.cdt.internal.core.index.IDocument;
+
+/**
+ * An indexedFile associates a number to a document path, and document properties. 
+ * It is what we add into an index, and the result of a query.
+ */
+
+public class IndexedFile implements IQueryResult {
+	protected String path;
+	protected int fileNumber;
+
+	public IndexedFile(String path, int fileNum) {
+		if (fileNum < 1)
+			throw new IllegalArgumentException();
+		this.fileNumber= fileNum;
+		this.path= path;
+	}
+	public IndexedFile(IDocument document, int fileNum) {
+		if (fileNum < 1)
+			throw new IllegalArgumentException();
+		this.path= document.getName();
+		this.fileNumber= fileNum;
+	}
+	/**
+	 * Returns the size of the indexedFile.
+	 */
+	public int footprint() {
+		//object+ 2 slots + size of the string (header + 4 slots + char[])
+		return 8 + (2 * 4) + (8 + (4 * 4) + 8 + path.length() * 2);
+	}
+	/**
+	 * Returns the file number.
+	 */
+	public int getFileNumber() {
+		return fileNumber;
+	}
+	/**
+	 * Returns the path.
+	 */
+	public String getPath() {
+		return path;
+	}
+	/**
+	 * Sets the file number.
+	 */
+	public void setFileNumber(int fileNumber) {
+		this.fileNumber= fileNumber;
+	}
+	public String toString() {
+		return "IndexedFile(" + fileNumber + ": " + path + ")"; //$NON-NLS-2$ //$NON-NLS-1$ //$NON-NLS-3$
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/IndexedFileHashedArray.java

@@ -0,0 +1,109 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.util.ArrayList;
+
+import org.eclipse.cdt.internal.core.index.IDocument;
+
+public final class IndexedFileHashedArray {
+
+private IndexedFile elements[];
+private int elementSize; // number of elements in the table
+private int threshold;
+private int lastId;
+private ArrayList replacedElements;
+
+public IndexedFileHashedArray(int size) {
+	if (size < 7) size = 7;
+	this.elements = new IndexedFile[2 * size + 1];
+	this.elementSize = 0;
+	this.threshold = size + 1; // size is the expected number of elements
+	this.lastId = 0;
+	this.replacedElements = null;
+}
+
+public IndexedFile add(IDocument document) {
+	return add(new IndexedFile(document, ++lastId));
+}
+
+private IndexedFile add(IndexedFile file) {
+	int length = elements.length;
+	String path = file.getPath();
+	int index = (path.hashCode() & 0x7FFFFFFF) % length;
+	IndexedFile current;
+	while ((current = elements[index]) != null) {
+		if (current.getPath().equals(path)) {
+			if (replacedElements == null) replacedElements = new ArrayList(5);
+			replacedElements.add(current);
+			return elements[index] = file;
+		}
+		if (++index == length) index = 0;
+	}
+	elements[index] = file;
+
+	// assumes the threshold is never equal to the size of the table
+	if (++elementSize > threshold) grow();
+	return file;
+}
+
+public IndexedFile[] asArray() {
+	IndexedFile[] array = new IndexedFile[lastId];
+	for (int i = 0, length = elements.length; i < length; i++) {
+		IndexedFile current = elements[i];
+		if (current != null)
+			array[current.fileNumber - 1] = current;
+	}
+	if (replacedElements != null) {
+		for (int i = 0, length = replacedElements.size(); i < length; i++) {
+			IndexedFile current = (IndexedFile) replacedElements.get(i);
+			array[current.fileNumber - 1] = current;
+		}
+	}
+	return array;
+}
+
+public IndexedFile get(String path) {
+	int length = elements.length;
+	int index = (path.hashCode() & 0x7FFFFFFF) % length;
+	IndexedFile current;
+	while ((current = elements[index]) != null) {
+		if (current.getPath().equals(path)) return current;
+		if (++index == length) index = 0;
+	}
+	return null;
+}
+
+private void grow() {
+	IndexedFileHashedArray newArray = new IndexedFileHashedArray(elementSize * 2); // double the number of expected elements
+	for (int i = 0, length = elements.length; i < length; i++)
+		if (elements[i] != null)
+			newArray.add(elements[i]);
+
+	// leave replacedElements as is
+	this.elements = newArray.elements;
+	this.elementSize = newArray.elementSize;
+	this.threshold = newArray.threshold;
+}
+
+public int size() {
+	return elementSize + (replacedElements == null ? 0 : replacedElements.size());
+}
+
+public String toString() {
+	String s = ""; //$NON-NLS-1$
+	IndexedFile[] files = asArray();
+	for (int i = 0, length = files.length; i < length; i++)
+		s += files[i].toString() + "\n"; 	//$NON-NLS-1$
+	return s;
+}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/IndexerOutput.java

@@ -0,0 +1,58 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import org.eclipse.cdt.internal.core.index.IIndexerOutput;
+import org.eclipse.cdt.internal.core.index.IDocument;
+
+/**
+ * An indexerOutput is used by an indexer to add documents and word references to
+ * an inMemoryIndex. It keeps track of the document being indexed and add the
+ * word references to this document (so you do not need to precise the document
+ * each time you add a word).
+ */
+
+public class IndexerOutput implements IIndexerOutput {
+	protected InMemoryIndex index;
+	protected IndexedFile indexedFile;
+	protected IDocument document;
+	/**
+	 * IndexerOutput constructor comment.
+	 */
+	public IndexerOutput(InMemoryIndex index) {
+		this.index= index;
+	}
+	/**
+	 * Adds the given document to the inMemoryIndex.
+	 */
+	public void addDocument(IDocument document) {
+		if (indexedFile == null) {
+			indexedFile= index.addDocument(document);
+		} else {
+			throw new IllegalStateException();
+		}
+	}
+	/**
+	 * Adds a reference to the given word to the inMemoryIndex.
+	 */
+	public void addRef(char[] word) {
+		if (indexedFile == null) {
+			throw new IllegalStateException();
+		}
+		index.addRef(indexedFile, word);
+	}
+	/**
+	 * Adds a reference to the given word to the inMemoryIndex.
+	 */
+	public void addRef(String word) {
+		addRef(word.toCharArray());
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/Int.java

@@ -0,0 +1,21 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+public class Int {
+	public int value;
+	/**
+	 * Int constructor comment.
+	 */
+	public Int(int i) {
+		value= i;
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/MergeFactory.java

@@ -0,0 +1,225 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * A mergeFactory is used to merge 2 indexes into one. One of the indexes 
+ * (oldIndex) is on the disk and the other(addsIndex) is in memory.
+ * The merge respects the following rules: <br>
+ *   - The files are sorted in alphabetical order;<br>
+ *   - if a file is in oldIndex and addsIndex, the one which is added 
+ * is the one in the addsIndex.<br>
+ */
+public class MergeFactory {
+	/**
+	 * Input on the addsIndex.
+	 */
+	protected IndexInput addsInput;
+	/**
+	 * Input on the oldIndex. 
+	 */
+	protected IndexInput oldInput;
+	/**
+	 * Output to write the result of the merge in.
+	 */
+	protected BlocksIndexOutput mergeOutput;
+	/**
+	 * Files removed from oldIndex. 
+	 */
+	protected Map removedInOld;
+	/**
+	 * Files removed from addsIndex. 
+	 */
+	protected Map removedInAdds;
+	protected int[] mappingOld;
+	protected int[] mappingAdds;
+	public static final int ADDS_INDEX= 0;
+	public static final int OLD_INDEX= 1;
+	/**
+	 * MergeFactory constructor comment.
+	 * @param directory java.io.File
+	 */
+	public MergeFactory(IndexInput oldIndexInput, IndexInput addsIndexInput, BlocksIndexOutput mergeIndexOutput, Map removedInOld, Map removedInAdds) {
+		oldInput= oldIndexInput;
+		addsInput= addsIndexInput;
+		mergeOutput= mergeIndexOutput;
+		this.removedInOld= removedInOld;
+		this.removedInAdds= removedInAdds;
+	}
+	/**
+	 * Initialise the merge.
+	 */
+	protected void init() {
+		mappingOld= new int[oldInput.getNumFiles() + 1];
+		mappingAdds= new int[addsInput.getNumFiles() + 1];
+
+	}
+	/**
+	 * Merges the 2 indexes into a new one on the disk.
+	 */
+	public void merge() throws IOException {
+		try {
+			//init
+			addsInput.open();
+			oldInput.open();
+			mergeOutput.open();
+			init();
+			//merge
+			//findChanges();
+			mergeFiles();
+			mergeReferences();
+			mergeOutput.flush();
+		} finally {
+			//closes everything
+			oldInput.close();
+			addsInput.close();
+			mergeOutput.close();
+		}
+	}
+	/**
+	 * Merges the files of the 2 indexes in the new index, removes the files
+	 * to be removed, and records the changes made to propagate them to the 
+	 * word references.
+	 */
+
+	protected void mergeFiles() throws IOException {
+		int positionInMerge= 1;
+		int compare;
+		while (oldInput.hasMoreFiles() || addsInput.hasMoreFiles()) {
+			IndexedFile file1= oldInput.getCurrentFile();
+			IndexedFile file2= addsInput.getCurrentFile();
+
+			//if the file has been removed we don't take it into account
+			while (file1 != null && wasRemoved(file1, OLD_INDEX)) {
+				oldInput.moveToNextFile();
+				file1= oldInput.getCurrentFile();
+			}
+			while (file2 != null && wasRemoved(file2, ADDS_INDEX)) {
+				addsInput.moveToNextFile();
+				file2= addsInput.getCurrentFile();
+			}
+
+			//the addsIndex was empty, we just removed files from the oldIndex
+			if (file1 == null && file2 == null)
+				break;
+
+			//test if we reached the end of one the 2 index
+			if (file1 == null)
+				compare= 1;
+			else if (file2 == null)
+				compare= -1;
+			else
+				compare= file1.getPath().compareTo(file2.getPath());
+
+			//records the changes to Make
+			if (compare == 0) {
+				//the file has been modified: 
+				//we remove it from the oldIndex and add it to the addsIndex
+				removeFile(file1, OLD_INDEX);
+				mappingAdds[file2.getFileNumber()]= positionInMerge;
+				file1.setFileNumber(positionInMerge);
+				mergeOutput.addFile(file1);
+				oldInput.moveToNextFile();
+				addsInput.moveToNextFile();
+			} else if (compare < 0) {
+				mappingOld[file1.getFileNumber()]= positionInMerge;
+				file1.setFileNumber(positionInMerge);
+				mergeOutput.addFile(file1);
+				oldInput.moveToNextFile();
+			} else {
+				mappingAdds[file2.getFileNumber()]= positionInMerge;
+				file2.setFileNumber(positionInMerge);
+				mergeOutput.addFile(file2);
+				addsInput.moveToNextFile();
+			}
+			positionInMerge++;
+		}
+		mergeOutput.flushFiles();		
+	}
+	/**
+	 * Merges the files of the 2 indexes in the new index, according to the changes
+	 * recorded during mergeFiles().
+	 */
+	protected void mergeReferences() throws IOException {
+		int compare;
+		while (oldInput.hasMoreWords() || addsInput.hasMoreWords()) {
+			WordEntry word1= oldInput.getCurrentWordEntry();
+			WordEntry word2= addsInput.getCurrentWordEntry();
+
+			if (word1 == null && word2 == null)
+				break;
+			
+			if (word1 == null)
+				compare= 1;
+			else if (word2 == null)
+				compare= -1;
+			else
+				compare= Util.compare(word1.getWord(), word2.getWord());
+			if (compare < 0) {
+				word1.mapRefs(mappingOld);
+				mergeOutput.addWord(word1);
+				oldInput.moveToNextWordEntry();
+			} else if (compare > 0) {
+				word2.mapRefs(mappingAdds);
+				mergeOutput.addWord(word2);
+				addsInput.moveToNextWordEntry();
+			} else {
+				word1.mapRefs(mappingOld);
+				word2.mapRefs(mappingAdds);
+				word1.addRefs(word2.getRefs());
+				mergeOutput.addWord(word1);
+				addsInput.moveToNextWordEntry();
+				oldInput.moveToNextWordEntry();
+			}
+		}
+		mergeOutput.flushWords();
+	}
+	/**
+	 * Records the deletion of one file.
+	 */
+	protected void removeFile(IndexedFile file, int index) {
+		if (index == OLD_INDEX)
+			mappingOld[file.getFileNumber()]= -1;
+		else
+			mappingAdds[file.getFileNumber()]= -1;
+	}
+	/**
+	 * Returns whether the given file has to be removed from the given index
+	 * (ADDS_INDEX or OLD_INDEX). If it has to be removed, the mergeFactory 
+	 * deletes it and records the changes. 
+	 */
+
+	protected boolean wasRemoved(IndexedFile indexedFile, int index) {
+		String path= indexedFile.getPath();
+		if (index == OLD_INDEX) {
+			if (removedInOld.remove(path) != null) {
+				mappingOld[indexedFile.getFileNumber()]= -1;
+				return true;
+			}
+		} else if (index == ADDS_INDEX) {
+			Int lastRemoved= (Int) removedInAdds.get(path);
+			if (lastRemoved != null) {
+				int fileNum= indexedFile.getFileNumber();
+				if (lastRemoved.value >= fileNum) {
+					mappingAdds[fileNum]= -1;
+					//if (lastRemoved.value == fileNum) // ONLY if files in sorted order for names AND fileNums
+					//removedInAdds.remove(path);
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/PropertyDocument.java

@@ -0,0 +1,48 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.util.Enumeration;
+import java.util.Hashtable;
+
+import org.eclipse.cdt.internal.core.index.IDocument;
+
+/**
+ * The properties of a document are stored into a hashtable.
+ * @see IDocument
+ */
+
+public abstract class PropertyDocument implements IDocument {
+	protected Hashtable properties;
+	public PropertyDocument() {
+		properties= new Hashtable(5);
+	}
+	/**
+	 * @see IDocument#getProperty
+	 */
+	public String getProperty(String property) {
+		return (String) properties.get(property);
+	}
+	/**
+	 * @see IDocument#getPropertyNames
+	 */
+
+	public Enumeration getPropertyNames() {
+		return properties.keys();
+	}
+	/**
+	 * @see IDocument#setProperty
+	 */
+
+	public void setProperty(String property, String value) {
+		properties.put(property, value);
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/SafeRandomAccessFile.java

@@ -0,0 +1,31 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+/**
+ * A safe subclass of RandomAccessFile, which ensure that it's closed
+ * on finalize.
+ */
+public class SafeRandomAccessFile extends RandomAccessFile {
+	public SafeRandomAccessFile(java.io.File file, String mode) throws java.io.IOException {
+		super(file, mode);
+	}
+	public SafeRandomAccessFile(String name, String mode) throws java.io.IOException {
+		super(name, mode);
+	}
+	protected void finalize() throws IOException {
+		close();
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/SimpleIndexInput.java

@@ -0,0 +1,177 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.eclipse.cdt.internal.core.index.IDocument;
+import org.eclipse.cdt.internal.core.index.IEntryResult;
+import org.eclipse.cdt.internal.core.index.IQueryResult;
+
+/**
+ * A simpleIndexInput is an input on an in memory Index. 
+ */
+
+public class SimpleIndexInput extends IndexInput {
+	protected WordEntry[] sortedWordEntries;
+	protected IndexedFile currentFile;
+	protected IndexedFile[] sortedFiles;
+	protected InMemoryIndex index;
+
+	public SimpleIndexInput(InMemoryIndex index) {
+		super();
+		this.index= index;
+	}
+	/**
+	 * @see IndexInput#clearCache()
+	 */
+	public void clearCache() {
+	}
+	/**
+	 * @see IndexInput#close()
+	 */
+	public void close() throws IOException {
+		sortedFiles= null;
+	}
+	/**
+	 * @see IndexInput#getCurrentFile()
+	 */
+	public IndexedFile getCurrentFile() throws IOException {
+		if (!hasMoreFiles())
+			return null;
+		return currentFile;
+	}
+	/**
+	 * @see IndexInput#getIndexedFile(int)
+	 */
+	public IndexedFile getIndexedFile(int fileNum) throws IOException {
+		for (int i= 0; i < sortedFiles.length; i++)
+			if (sortedFiles[i].getFileNumber() == fileNum)
+				return sortedFiles[i];
+		return null;
+	}
+	/**
+	 * @see IndexInput#getIndexedFile(IDocument)
+	 */
+	public IndexedFile getIndexedFile(IDocument document) throws IOException {
+		String name= document.getName();
+		for (int i= index.getNumFiles(); i >= 1; i--) {
+			IndexedFile file= getIndexedFile(i);
+			if (name.equals(file.getPath()))
+				return file;
+		}
+		return null;
+	}
+	/**
+	 * @see IndexInput#getNumFiles()
+	 */
+	public int getNumFiles() {
+		return index.getNumFiles();
+	}
+	/**
+	 * @see IndexInput#getNumWords()
+	 */
+	public int getNumWords() {
+		return sortedWordEntries.length;
+	}
+	/**
+	 * @see IndexInput#getSource()
+	 */
+	public Object getSource() {
+		return index;
+	}
+	public void init() {
+		index.init();
+
+	}
+	/**
+	 * @see IndexInput#moveToNextFile()
+	 */
+	public void moveToNextFile() throws IOException {
+		filePosition++;
+		if (!hasMoreFiles()) {
+			return;
+		}
+		currentFile= sortedFiles[filePosition - 1];
+	}
+	/**
+	 * @see IndexInput#moveToNextWordEntry()
+	 */
+	public void moveToNextWordEntry() throws IOException {
+		wordPosition++;
+		if (hasMoreWords())
+			currentWordEntry= sortedWordEntries[wordPosition - 1];
+	}
+	/**
+	 * @see IndexInput#open()
+	 */
+	public void open() throws IOException {
+		sortedWordEntries= index.getSortedWordEntries();
+		sortedFiles= index.getSortedFiles();
+		filePosition= 1;
+		wordPosition= 1;
+		setFirstFile();
+		setFirstWord();
+	}
+	/**
+	 * @see IndexInput#query(String)
+	 */
+	public IQueryResult[] query(String word) throws IOException {
+		char[] wordChar= word.toCharArray();
+		WordEntry wordEntry= index.getWordEntry(wordChar);
+		int[] fileNums= wordEntry.getRefs();
+		IQueryResult[] files= new IQueryResult[fileNums.length];
+		for (int i= 0; i < files.length; i++)
+			files[i]= getIndexedFile(fileNums[i]);
+		return files;
+	}
+	public IEntryResult[] queryEntriesPrefixedBy(char[] prefix) throws IOException {
+		return null;
+	}
+	public IQueryResult[] queryFilesReferringToPrefix(char[] prefix) throws IOException {
+			return null;
+	}
+	/**
+	 * @see IndexInput#queryInDocumentNames(String)
+	 */
+	public IQueryResult[] queryInDocumentNames(String word) throws IOException {
+		setFirstFile();
+		ArrayList matches= new ArrayList();
+		while (hasMoreFiles()) {
+			IndexedFile file= getCurrentFile();
+			if (file.getPath().indexOf(word) != -1)
+				matches.add(file.getPath());
+			moveToNextFile();
+		}
+		IQueryResult[] match= new IQueryResult[matches.size()];
+		matches.toArray(match);
+		return match;
+	}
+	/**
+	 * @see IndexInput#setFirstFile()
+	 */
+	protected void setFirstFile() throws IOException {
+		filePosition= 1;
+		if (sortedFiles.length > 0) {
+			currentFile= sortedFiles[0];
+		}
+	}
+	/**
+	 * @see IndexInput#setFirstWord()
+	 */
+	protected void setFirstWord() throws IOException {
+		wordPosition= 1;
+		if (sortedWordEntries.length > 0)
+			currentWordEntry= sortedWordEntries[0];
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/Util.java

@@ -0,0 +1,341 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UTFDataFormatException;
+
+public class Util {
+
+	private Util() {
+	}
+	/**
+	 * Compares two strings lexicographically. 
+	 * The comparison is based on the Unicode value of each character in
+	 * the strings. 
+	 *
+	 * @return  the value <code>0</code> if the str1 is equal to str2;
+	 *          a value less than <code>0</code> if str1
+	 *          is lexicographically less than str2; 
+	 *          and a value greater than <code>0</code> if str1 is
+	 *          lexicographically greater than str2.
+	 */
+	public static int compare(char[] str1, char[] str2) {
+		int len1= str1.length;
+		int len2= str2.length;
+		int n= Math.min(len1, len2);
+		int i= 0;
+		while (n-- != 0) {
+			char c1= str1[i];
+			char c2= str2[i++];
+			if (c1 != c2) {
+				return c1 - c2;
+			}
+		}
+		return len1 - len2;
+	}
+
+	/**
+	 * Returns the length of the common prefix between s1 and s2.
+	 */
+	public static int prefixLength(char[] s1, char[] s2) {
+		int len= 0;
+		int max= Math.min(s1.length, s2.length);
+		for (int i= 0; i < max && s1[i] == s2[i]; ++i)
+			++len;
+		return len;
+	}
+	/**
+	 * Returns the length of the common prefix between s1 and s2.
+	 */
+	public static int prefixLength(String s1, String s2) {
+		int len= 0;
+		int max= Math.min(s1.length(), s2.length());
+		for (int i= 0; i < max && s1.charAt(i) == s2.charAt(i); ++i)
+			++len;
+		return len;
+	}
+	private static void quickSort(char[][] list, int left, int right) {
+		int original_left= left;
+		int original_right= right;
+		char[] mid= list[(left + right) / 2];
+		do {
+			while (compare(list[left], mid) < 0) {
+				left++;
+			}
+			while (compare(mid, list[right]) < 0) {
+				right--;
+			}
+			if (left <= right) {
+				char[] tmp= list[left];
+				list[left]= list[right];
+				list[right]= tmp;
+				left++;
+				right--;
+			}
+		} while (left <= right);
+		if (original_left < right) {
+			quickSort(list, original_left, right);
+		}
+		if (left < original_right) {
+			quickSort(list, left, original_right);
+		}
+	}
+	private static void quickSort(int[] list, int left, int right) {
+		int original_left= left;
+		int original_right= right;
+		int mid= list[(left + right) / 2];
+		do {
+			while (list[left] < mid) {
+				left++;
+			}
+			while (mid < list[right]) {
+				right--;
+			}
+			if (left <= right) {
+				int tmp= list[left];
+				list[left]= list[right];
+				list[right]= tmp;
+				left++;
+				right--;
+			}
+		} while (left <= right);
+		if (original_left < right) {
+			quickSort(list, original_left, right);
+		}
+		if (left < original_right) {
+			quickSort(list, left, original_right);
+		}
+	}
+	private static void quickSort(String[] list, int left, int right) {
+		int original_left= left;
+		int original_right= right;
+		String mid= list[(left + right) / 2];
+		do {
+			while (list[left].compareTo(mid) < 0) {
+				left++;
+			}
+			while (mid.compareTo(list[right]) < 0) {
+				right--;
+			}
+			if (left <= right) {
+				String tmp= list[left];
+				list[left]= list[right];
+				list[right]= tmp;
+				left++;
+				right--;
+			}
+		} while (left <= right);
+		if (original_left < right) {
+			quickSort(list, original_left, right);
+		}
+		if (left < original_right) {
+			quickSort(list, left, original_right);
+		}
+	}
+	private static void quickSort(IndexedFile[] list, int left, int right) {
+		int original_left= left;
+		int original_right= right;
+		String mid= list[(left + right) / 2].path;
+		do {
+			while (list[left].path.compareTo(mid) < 0) {
+				left++;
+			}
+			while (mid.compareTo(list[right].path) < 0) {
+				right--;
+			}
+			if (left <= right) {
+				IndexedFile tmp= list[left];
+				list[left]= list[right];
+				list[right]= tmp;
+				left++;
+				right--;
+			}
+		} while (left <= right);
+		if (original_left < right) {
+			quickSort(list, original_left, right);
+		}
+		if (left < original_right) {
+			quickSort(list, left, original_right);
+		}
+	}
+	private static void quickSort(WordEntry[] list, int left, int right) {
+		int original_left= left;
+		int original_right= right;
+		char[] mid= list[(left + right) / 2].fWord;
+		do {
+			while (compare(list[left].fWord, mid) < 0) {
+				left++;
+			}
+			while (compare(mid, list[right].fWord) < 0) {
+				right--;
+			}
+			if (left <= right) {
+				WordEntry tmp= list[left];
+				list[left]= list[right];
+				list[right]= tmp;
+				left++;
+				right--;
+			}
+		} while (left <= right);
+		if (original_left < right) {
+			quickSort(list, original_left, right);
+		}
+		if (left < original_right) {
+			quickSort(list, left, original_right);
+		}
+	}
+	/**
+	 * Reads in a string from the specified data input stream. The 
+	 * string has been encoded using a modified UTF-8 format. 
+	 * <p>
+	 * The first two bytes are read as if by 
+	 * <code>readUnsignedShort</code>. This value gives the number of 
+	 * following bytes that are in the encoded string, not
+	 * the length of the resulting string. The following bytes are then 
+	 * interpreted as bytes encoding characters in the UTF-8 format 
+	 * and are converted into characters. 
+	 * <p>
+	 * This method blocks until all the bytes are read, the end of the 
+	 * stream is detected, or an exception is thrown. 
+	 *
+	 * @param      in   a data input stream.
+	 * @return     a Unicode string.
+	 * @exception  EOFException            if the input stream reaches the end
+	 *               before all the bytes.
+	 * @exception  IOException             if an I/O error occurs.
+	 * @exception  UTFDataFormatException  if the bytes do not represent a
+	 *               valid UTF-8 encoding of a Unicode string.
+	 * @see        java.io.DataInputStream#readUnsignedShort()
+	 */
+	public final static char[] readUTF(DataInput in) throws IOException {
+		int utflen= in.readUnsignedShort();
+		char str[]= new char[utflen];
+		int count= 0;
+		int strlen= 0;
+		while (count < utflen) {
+			int c= in.readUnsignedByte();
+			int char2, char3;
+			switch (c >> 4) {
+				case 0 :
+				case 1 :
+				case 2 :
+				case 3 :
+				case 4 :
+				case 5 :
+				case 6 :
+				case 7 :
+					// 0xxxxxxx
+					count++;
+					str[strlen++]= (char) c;
+					break;
+				case 12 :
+				case 13 :
+					// 110x xxxx   10xx xxxx
+					count += 2;
+					if (count > utflen)
+						throw new UTFDataFormatException();
+					char2= in.readUnsignedByte();
+					if ((char2 & 0xC0) != 0x80)
+						throw new UTFDataFormatException();
+					str[strlen++]= (char) (((c & 0x1F) << 6) | (char2 & 0x3F));
+					break;
+				case 14 :
+					// 1110 xxxx  10xx xxxx  10xx xxxx
+					count += 3;
+					if (count > utflen)
+						throw new UTFDataFormatException();
+					char2= in.readUnsignedByte();
+					char3= in.readUnsignedByte();
+					if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80))
+						throw new UTFDataFormatException();
+					str[strlen++]= (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | ((char3 & 0x3F) << 0));
+					break;
+				default :
+					// 10xx xxxx,  1111 xxxx
+					throw new UTFDataFormatException();
+			}
+		}
+		if (strlen < utflen) {
+			System.arraycopy(str, 0, str= new char[strlen], 0, strlen);
+		}
+		return str;
+	}
+	public static void sort(char[][] list) {
+		if (list.length > 1)
+			quickSort(list, 0, list.length - 1);
+	}
+	public static void sort(int[] list) {
+		if (list.length > 1)
+			quickSort(list, 0, list.length - 1);
+	}
+	public static void sort(String[] list) {
+		if (list.length > 1)
+			quickSort(list, 0, list.length - 1);
+	}
+	public static void sort(IndexedFile[] list) {
+		if (list.length > 1)
+			quickSort(list, 0, list.length - 1);
+	}
+	public static void sort(WordEntry[] list) {
+		if (list.length > 1)
+			quickSort(list, 0, list.length - 1);
+	}
+	/**
+	 * Writes a string to the given output stream using UTF-8 
+	 * encoding in a machine-independent manner. 
+	 * <p>
+	 * First, two bytes are written to the output stream as if by the 
+	 * <code>writeShort</code> method giving the number of bytes to 
+	 * follow. This value is the number of bytes actually written out, 
+	 * not the length of the string. Following the length, each character 
+	 * of the string is output, in sequence, using the UTF-8 encoding 
+	 * for the character. 
+	 *
+	 * @param      str   a string to be written.
+	 * @exception  IOException  if an I/O error occurs.
+	 * @since      JDK1.0
+	 */
+	public static void writeUTF(OutputStream out, char[] str) throws IOException {
+		int strlen= str.length;
+		int utflen= 0;
+		for (int i= 0; i < strlen; i++) {
+			int c= str[i];
+			if ((c >= 0x0001) && (c <= 0x007F)) {
+				utflen++;
+			} else if (c > 0x07FF) {
+				utflen += 3;
+			} else {
+				utflen += 2;
+			}
+		}
+		if (utflen > 65535)
+			throw new UTFDataFormatException();
+		out.write((utflen >>> 8) & 0xFF);
+		out.write((utflen >>> 0) & 0xFF);
+		for (int i= 0; i < strlen; i++) {
+			int c= str[i];
+			if ((c >= 0x0001) && (c <= 0x007F)) {
+				out.write(c);
+			} else if (c > 0x07FF) {
+				out.write(0xE0 | ((c >> 12) & 0x0F));
+				out.write(0x80 | ((c >> 6) & 0x3F));
+				out.write(0x80 | ((c >> 0) & 0x3F));
+			} else {
+				out.write(0xC0 | ((c >> 6) & 0x1F));
+				out.write(0x80 | ((c >> 0) & 0x3F));
+			}
+		}
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/WordEntry.java

@@ -0,0 +1,156 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import org.eclipse.cdt.internal.core.search.CharOperation;
+
+public class WordEntry {
+	protected char[] fWord;
+	protected int fNumRefs;
+	protected int[] fRefs;
+	public WordEntry() {
+		this(CharOperation.NO_CHAR);
+	}
+	public WordEntry(char[] word) {
+		fWord= word;
+		fNumRefs= 0;
+		fRefs= new int[1];
+	}
+	/**
+	 * Adds a reference and records the change in footprint.
+	 */
+	public int addRef(int fileNum) {
+		if (fNumRefs > 0 && fRefs[fNumRefs - 1] == fileNum) {
+			return 0;
+		}
+		if (fNumRefs < fRefs.length) {
+			fRefs[fNumRefs++]= fileNum;
+			return 0;
+		} 
+
+		// For rt.jar, 73265 word entries are created. 51997 have 1 ref, then 9438, 3738, 1980, 1214, 779, 547, 429, 371 etc.
+		int newSize= fNumRefs < 4 ? 4 : fNumRefs * 2; // so will start @ 1, grow to 4, 8, 16, 32, 64 etc.
+		System.arraycopy(fRefs, 0, fRefs= new int[newSize], 0, fNumRefs);
+		fRefs[fNumRefs++]= fileNum;
+		return (newSize - fNumRefs + 1) * 4;
+	}
+	/**
+	 * Adds a set of references and records the change in footprint.
+	 */
+	public void addRefs(int[] refs) {
+		int[] newRefs= new int[fNumRefs + refs.length];
+		int pos1= 0;
+		int pos2= 0;
+		int posNew= 0;
+		int compare;
+		int r1= 0;
+		int r2= 0;
+		while (pos1 < fNumRefs || pos2 < refs.length) {
+			if (pos1 >= fNumRefs) {
+				r2= refs[pos2];
+				compare= -1;
+			} else if (pos2 >= refs.length) {
+				compare= 1;
+				r1= fRefs[pos1];
+			} else {
+				r1= fRefs[pos1];
+				r2= refs[pos2];
+				compare= r2 - r1;
+			}
+			if (compare > 0) {
+				newRefs[posNew]= r1;
+				posNew++;
+				pos1++;
+			} else {
+				if (r2 != 0) {
+					newRefs[posNew]= r2;
+					posNew++;
+				}
+				pos2++;
+			}
+		}
+		fRefs= newRefs;
+		fNumRefs= posNew;
+		/*for (int i = 0; i < refs.length; i++)
+		addRef(refs[i]);
+		int[] newRefs = new int[fNumRefs];
+		System.arraycopy(fRefs, 0, newRefs, 0, fNumRefs);
+		fRefs = newRefs;
+		Util.sort(fRefs);*/
+	}
+	/**
+	 * Returns the size of the wordEntry
+	 */
+	public int footprint() {
+		return 8 + (3 * 4) + (8 + fWord.length * 2) + (8 + fRefs.length * 4);
+	}
+	/**
+	 * Returns the number of references, e.g. the number of files this word appears in.
+	 */
+	public int getNumRefs() {
+		return fNumRefs;
+	}
+	/**
+	 * returns the file number in the i position in the list of references.
+	 */
+	public int getRef(int i) {
+		if (i < fNumRefs) return fRefs[i];
+		throw new IndexOutOfBoundsException();
+	}
+	/**
+	 * Returns the references of the wordEntry (the number of the files it appears in).
+	 */
+	public int[] getRefs() {
+		int[] result= new int[fNumRefs];
+		System.arraycopy(fRefs, 0, result, 0, fNumRefs);
+		return result;
+	}
+	/**
+	 * returns the word of the wordEntry.
+	 */
+	public char[] getWord() {
+		return fWord;
+	}
+	/**
+	 * Changes the references of the wordEntry to match the mapping. For example,<br>
+	 * if the current references are [1 3 4]<br>
+	 * and mapping is [1 2 3 4 5]<br>
+	 * in references 1 becomes mapping[1] = 2, 3->4, and 4->5<br>
+	 * => references = [2 4 5].<br>
+	 */
+	public void mapRefs(int[] mappings) {
+		int position= 0;
+		for (int i= 0; i < fNumRefs; i++) {
+			int map= mappings[fRefs[i]];
+			if (map != -1 && map != 0)
+				fRefs[position++]= map;
+		}
+		fNumRefs= position;
+
+		//to be changed!
+		System.arraycopy(fRefs, 0, (fRefs= new int[fNumRefs]), 0, fNumRefs);
+		Util.sort(fRefs);
+	}
+	/**
+	 * Clears the wordEntry.
+	 */
+	public void reset(char[] word) {
+		for (int i= fNumRefs; i-- > 0;) {
+			fRefs[i]= 0;
+		}
+		fNumRefs= 0;
+		fWord= word;
+	}
+	public String toString() {
+		return new String(fWord);
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/index/impl/WordEntryHashedArray.java

@@ -0,0 +1,83 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.index.impl;
+
+import org.eclipse.cdt.internal.core.search.CharOperation;
+
+public final class WordEntryHashedArray {
+	
+	// to avoid using Enumerations, walk the objects skipping nulls
+	public WordEntry elements[];
+	public int elementSize; // number of elements in the table
+	public int threshold;
+	
+	public WordEntryHashedArray(int size) {
+		if (size < 7) size = 7;
+		this.elements = new WordEntry[2 * size + 1];
+		this.elementSize = 0;
+		this.threshold = size + 1; // size is the expected number of elements
+	}
+	
+	public WordEntry add(WordEntry entry) {
+		int length = elements.length;
+		char[] word = entry.getWord();
+		int index = CharOperation.hashCode(word) % length;
+		WordEntry current;
+		while ((current = elements[index]) != null) {
+			if (CharOperation.equals(current.getWord(), word)) return elements[index] = entry;
+			if (++index == length) index = 0;
+		}
+		elements[index] = entry;
+	
+		// assumes the threshold is never equal to the size of the table
+		if (++elementSize > threshold) grow();
+		return entry;
+	}
+	
+	public WordEntry[] asArray() {
+		WordEntry[] array = new WordEntry[elementSize];
+		for (int i = 0, j = 0, length = elements.length; i < length; i++) {
+			WordEntry current = elements[i];
+			if (current != null) array[j++] = current;
+		}
+		return array;
+	}
+	
+	public WordEntry get(char[] word) {
+		int length = elements.length;
+		int index = CharOperation.hashCode(word) % length;
+		WordEntry current;
+		while ((current = elements[index]) != null) {
+			if (CharOperation.equals(current.getWord(), word)) return current;
+			if (++index == length) index = 0;
+		}
+		return null;
+	}
+	
+	private void grow() {
+		WordEntryHashedArray newArray = new WordEntryHashedArray(elementSize * 2); // double the number of expected elements
+		for (int i = 0, length = elements.length; i < length; i++)
+			if (elements[i] != null)
+				newArray.add(elements[i]);
+	
+		this.elements = newArray.elements;
+		this.elementSize = newArray.elementSize;
+		this.threshold = newArray.threshold;
+	}
+	
+	public String toString() {
+		String s = ""; //$NON-NLS-1$
+		WordEntry[] entries = asArray();
+		for (int i = 0, length = entries.length; i < length; i++)
+			s += entries[i].toString() + "\n"; 	//$NON-NLS-1$
+		return s;
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/HashtableOfInt.java

@@ -0,0 +1,93 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.search;
+
+ /**
+  *	Hashtable for non-zero int keys.
+  */
+  
+public final class HashtableOfInt {
+	// to avoid using Enumerations, walk the individual tables skipping nulls
+	public int[] keyTable;
+	public Object[] valueTable;
+
+	int elementSize; // number of elements in the table
+	int threshold;
+	public HashtableOfInt() {
+		this(13);
+	}
+	public HashtableOfInt(int size) {
+		this.elementSize = 0;
+		this.threshold = size; // size represents the expected number of elements
+		int extraRoom = (int) (size * 1.75f);
+		if (this.threshold == extraRoom)
+			extraRoom++;
+		this.keyTable = new int[extraRoom];
+		this.valueTable = new Object[extraRoom];
+	}
+	public boolean containsKey(int key) {
+		int index = key % valueTable.length;
+		int currentKey;
+		while ((currentKey = keyTable[index]) != 0) {
+			if (currentKey == key)
+				return true;
+			index = (index + 1) % keyTable.length;
+		}
+		return false;
+	}
+	public Object get(int key) {
+		int index = key % valueTable.length;
+		int currentKey;
+		while ((currentKey = keyTable[index]) != 0) {
+			if (currentKey == key)  return valueTable[index];
+			index = (index + 1) % keyTable.length;
+		}
+		return null;
+	}
+	public Object put(int key, Object value) {
+		int index = key % valueTable.length;
+		int currentKey;
+		while ((currentKey = keyTable[index]) != 0) {
+			if (currentKey == key)  return valueTable[index] = value;
+			index = (index + 1) % keyTable.length;
+		}
+		keyTable[index] = key;
+		valueTable[index] = value;
+	
+		// assumes the threshold is never equal to the size of the table
+		if (++elementSize > threshold)
+			rehash();
+		return value;
+	}
+	private void rehash() {
+		HashtableOfInt newHashtable = new HashtableOfInt(elementSize * 2); // double the number of expected elements
+		int currentKey;
+		for (int i = keyTable.length; --i >= 0;)
+			if ((currentKey = keyTable[i]) != 0)
+				newHashtable.put(currentKey, valueTable[i]);
+	
+		this.keyTable = newHashtable.keyTable;
+		this.valueTable = newHashtable.valueTable;
+		this.threshold = newHashtable.threshold;
+	}
+	public int size() {
+		return elementSize;
+	}
+	public String toString() {
+		String s = ""; //$NON-NLS-1$
+		Object object;
+		for (int i = 0, length = valueTable.length; i < length; i++)
+			if ((object = valueTable[i]) != null)
+				s += keyTable[i] + " -> " + object.toString() + "\n"; //$NON-NLS-2$ //$NON-NLS-1$
+		return s;
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/SimpleLookupTable.java

@@ -0,0 +1,153 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.search;
+
+/**
+ * A simple lookup table is a non-synchronized Hashtable, whose keys
+ * and values are Objects. It also uses linear probing to resolve collisions
+ * rather than a linked list of hash table entries.
+ */
+public final class SimpleLookupTable implements Cloneable {
+	
+	// to avoid using Enumerations, walk the individual tables skipping nulls
+	public Object[] keyTable;
+	public Object[] valueTable;
+	public int elementSize; // number of elements in the table
+	public int threshold;
+	
+	public SimpleLookupTable() {
+		this(13);
+	}
+	
+	public SimpleLookupTable(int size) {
+		if (size < 3) size = 3;
+		this.elementSize = 0;
+		this.threshold = size + 1; // size is the expected number of elements
+		int tableLength = 2 * size + 1;
+		this.keyTable = new Object[tableLength];
+		this.valueTable = new Object[tableLength];
+	}
+	
+	public Object clone() throws CloneNotSupportedException {
+		SimpleLookupTable result = (SimpleLookupTable) super.clone();
+		result.elementSize = this.elementSize;
+		result.threshold = this.threshold;
+	
+		int length = this.keyTable.length;
+		result.keyTable = new Object[length];
+		System.arraycopy(this.keyTable, 0, result.keyTable, 0, length);
+	
+		length = this.valueTable.length;
+		result.valueTable = new Object[length];
+		System.arraycopy(this.valueTable, 0, result.valueTable, 0, length);
+		return result;
+	}
+	
+	public boolean containsKey(Object key) {
+		int length = keyTable.length;
+		int index = (key.hashCode() & 0x7FFFFFFF) % length;
+		Object currentKey;
+		while ((currentKey = keyTable[index]) != null) {
+			if (currentKey.equals(key)) return true;
+			if (++index == length) index = 0;
+		}
+		return false;
+	}
+	
+	public Object get(Object key) {
+		int length = keyTable.length;
+		int index = (key.hashCode() & 0x7FFFFFFF) % length;
+		Object currentKey;
+		while ((currentKey = keyTable[index]) != null) {
+			if (currentKey.equals(key)) return valueTable[index];
+			if (++index == length) index = 0;
+		}
+		return null;
+	}
+	
+	public Object keyForValue(Object valueToMatch) {
+		if (valueToMatch != null)
+			for (int i = 0, l = valueTable.length; i < l; i++)
+				if (valueToMatch.equals(valueTable[i]))
+					return keyTable[i];
+		return null;
+	}
+	
+	public Object put(Object key, Object value) {
+		int length = keyTable.length;
+		int index = (key.hashCode() & 0x7FFFFFFF) % length;
+		Object currentKey;
+		while ((currentKey = keyTable[index]) != null) {
+			if (currentKey.equals(key)) return valueTable[index] = value;
+			if (++index == length) index = 0;
+		}
+		keyTable[index] = key;
+		valueTable[index] = value;
+	
+		// assumes the threshold is never equal to the size of the table
+		if (++elementSize > threshold) rehash();
+		return value;
+	}
+	
+	public void removeKey(Object key) {
+		int length = keyTable.length;
+		int index = (key.hashCode() & 0x7FFFFFFF) % length;
+		Object currentKey;
+		while ((currentKey = keyTable[index]) != null) {
+			if (currentKey.equals(key)) {
+				elementSize--;
+				keyTable[index] = null;
+				valueTable[index] = null;
+				if (keyTable[index + 1 == length ? 0 : index + 1] != null)
+					rehash(); // only needed if a possible collision existed
+				return;
+			}
+			if (++index == length) index = 0;
+		}
+	}
+	
+	public void removeValue(Object valueToRemove) {
+		boolean rehash = false;
+		for (int i = 0, l = valueTable.length; i < l; i++) {
+			Object value = valueTable[i];
+			if (value != null && value.equals(valueToRemove)) {
+				elementSize--;
+				keyTable[i] = null;
+				valueTable[i] = null;
+				if (!rehash && keyTable[i + 1 == l ? 0 : i + 1] != null)
+					rehash = true; // only needed if a possible collision existed
+			}
+		}
+		if (rehash) rehash();
+	}
+	
+	private void rehash() {
+		SimpleLookupTable newLookupTable = new SimpleLookupTable(elementSize * 2); // double the number of expected elements
+		Object currentKey;
+		for (int i = keyTable.length; --i >= 0;)
+			if ((currentKey = keyTable[i]) != null)
+				newLookupTable.put(currentKey, valueTable[i]);
+	
+		this.keyTable = newLookupTable.keyTable;
+		this.valueTable = newLookupTable.valueTable;
+		this.elementSize = newLookupTable.elementSize;
+		this.threshold = newLookupTable.threshold;
+	}
+	
+	public String toString() {
+		String s = ""; //$NON-NLS-1$
+		Object object;
+		for (int i = 0, l = valueTable.length; i < l; i++)
+			if ((object = valueTable[i]) != null)
+				s += keyTable[i].toString() + " -> " + object.toString() + "\n"; 	//$NON-NLS-2$ //$NON-NLS-1$
+		return s;
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/Util.java

@@ -0,0 +1,374 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+
+package org.eclipse.cdt.internal.core.search;
+
+import java.util.Locale;
+import java.util.MissingResourceException;
+import java.util.ResourceBundle;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.HashSet;
+
+import org.eclipse.core.resources.IContainer;
+import org.eclipse.core.resources.IResource;
+import org.eclipse.core.runtime.IPath;
+
+public class Util {
+	
+	private final static char[] DOUBLE_QUOTES = "''".toCharArray(); //$NON-NLS-1$
+	private final static char[] SINGLE_QUOTE = "'".toCharArray(); //$NON-NLS-1$
+	private static final int DEFAULT_READING_SIZE = 8192;
+
+	/* Bundle containing messages */
+	protected static ResourceBundle bundle;
+	private final static String bundleName = "org.eclipse.cdt.internal.core.search.messages"; //$NON-NLS-1$
+	static {
+		relocalize();
+	}
+	/**
+	 * Lookup the message with the given ID in this catalog and bind its
+	 * substitution locations with the given strings.
+	 */
+	public static String bind(String id, String binding1, String binding2) {
+		return bind(id, new String[] {binding1, binding2});
+	}
+	/**
+	 * Lookup the message with the given ID in this catalog and bind its
+	 * substitution locations with the given string.
+	 */
+	public static String bind(String id, String binding) {
+		return bind(id, new String[] {binding});
+	}
+	/**
+	 * Lookup the message with the given ID in this catalog and bind its
+	 * substitution locations with the given string values.
+	 */
+	public static String bind(String id, String[] bindings) {
+		if (id == null)
+			return "No message available"; //$NON-NLS-1$
+		String message = null;
+		try {
+			message = bundle.getString(id);
+		} catch (MissingResourceException e) {
+			// If we got an exception looking for the message, fail gracefully by just returning
+			// the id we were looking for.  In most cases this is semi-informative so is not too bad.
+			return "Missing message: " + id + " in: " + bundleName; //$NON-NLS-2$ //$NON-NLS-1$
+		}
+		// for compatibility with MessageFormat which eliminates double quotes in original message
+		char[] messageWithNoDoubleQuotes =
+		CharOperation.replace(message.toCharArray(), DOUBLE_QUOTES, SINGLE_QUOTE);
+		message = new String(messageWithNoDoubleQuotes);
+	
+		if (bindings == null)
+			return message;
+		int length = message.length();
+		int start = -1;
+		int end = length;
+		StringBuffer output = new StringBuffer(80);
+		while (true) {
+			if ((end = message.indexOf('{', start)) > -1) {
+				output.append(message.substring(start + 1, end));
+				if ((start = message.indexOf('}', end)) > -1) {
+					int index = -1;
+					try {
+						index = Integer.parseInt(message.substring(end + 1, start));
+						output.append(bindings[index]);
+					} catch (NumberFormatException nfe) {
+						output.append(message.substring(end + 1, start + 1));
+					} catch (ArrayIndexOutOfBoundsException e) {
+						output.append("{missing " + Integer.toString(index) + "}"); //$NON-NLS-2$ //$NON-NLS-1$
+					}
+				} else {
+					output.append(message.substring(end, length));
+					break;
+				}
+			} else {
+				output.append(message.substring(start + 1, length));
+				break;
+			}
+		}
+		return output.toString();
+	}
+	/**
+	 * Lookup the message with the given ID in this catalog 
+	 */
+	public static String bind(String id) {
+		return bind(id, (String[])null);
+	}
+	/**
+	 * Creates a NLS catalog for the given locale.
+	 */
+	public static void relocalize() {
+		try  {		
+			bundle = ResourceBundle.getBundle(bundleName, Locale.getDefault());
+		} catch(MissingResourceException e) {
+			System.out.println("Missing resource : " + bundleName.replace('.', '/') + ".properties for locale " + Locale.getDefault()); //$NON-NLS-1$//$NON-NLS-2$
+			throw e;
+		}
+	}
+	
+	/**
+	 * Returns the contents of the given file as a byte array.
+	 * @throws IOException if a problem occured reading the file.
+	 */
+	public static byte[] getFileByteContent(File file) throws IOException {
+		InputStream stream = null;
+		try {
+			stream = new BufferedInputStream(new FileInputStream(file));
+			return getInputStreamAsByteArray(stream, (int) file.length());
+		} finally {
+			if (stream != null) {
+				try {
+					stream.close();
+				} catch (IOException e) {
+				}
+			}
+		}
+	}
+	/**
+	 * Returns the contents of the given file as a char array.
+	 * When encoding is null, then the platform default one is used
+	 * @throws IOException if a problem occured reading the file.
+	 */
+	public static char[] getFileCharContent(File file, String encoding) throws IOException {
+		InputStream stream = null;
+		try {
+			stream = new BufferedInputStream(new FileInputStream(file));
+			return Util.getInputStreamAsCharArray(stream, (int) file.length(), encoding);
+		} finally {
+			if (stream != null) {
+				try {
+					stream.close();
+				} catch (IOException e) {
+				}
+			}
+		}
+	}
+	
+	/**
+	 * Returns the given input stream's contents as a byte array.
+	 * If a length is specified (ie. if length != -1), only length bytes
+	 * are returned. Otherwise all bytes in the stream are returned.
+	 * Note this doesn't close the stream.
+	 * @throws IOException if a problem occured reading the stream.
+	 */
+	public static byte[] getInputStreamAsByteArray(InputStream stream, int length)
+		throws IOException {
+		byte[] contents;
+		if (length == -1) {
+			contents = new byte[0];
+			int contentsLength = 0;
+			int amountRead = -1;
+			do {
+				int amountRequested = Math.max(stream.available(), DEFAULT_READING_SIZE);  // read at least 8K
+				
+				// resize contents if needed
+				if (contentsLength + amountRequested > contents.length) {
+					System.arraycopy(
+						contents,
+						0,
+						contents = new byte[contentsLength + amountRequested],
+						0,
+						contentsLength);
+				}
+
+				// read as many bytes as possible
+				amountRead = stream.read(contents, contentsLength, amountRequested);
+
+				if (amountRead > 0) {
+					// remember length of contents
+					contentsLength += amountRead;
+				}
+			} while (amountRead != -1); 
+
+			// resize contents if necessary
+			if (contentsLength < contents.length) {
+				System.arraycopy(
+					contents,
+					0,
+					contents = new byte[contentsLength],
+					0,
+					contentsLength);
+			}
+		} else {
+			contents = new byte[length];
+			int len = 0;
+			int readSize = 0;
+			while ((readSize != -1) && (len != length)) {
+				// See PR 1FMS89U
+				// We record first the read size. In this case len is the actual read size.
+				len += readSize;
+				readSize = stream.read(contents, len, length - len);
+			}
+		}
+
+		return contents;
+	}
+
+	/**
+	 * Returns the given input stream's contents as a character array.
+	 * If a length is specified (ie. if length != -1), only length chars
+	 * are returned. Otherwise all chars in the stream are returned.
+	 * Note this doesn't close the stream.
+	 * @throws IOException if a problem occured reading the stream.
+	 */
+	public static char[] getInputStreamAsCharArray(InputStream stream, int length, String encoding)
+		throws IOException {
+		InputStreamReader reader = null;
+		reader = encoding == null
+					? new InputStreamReader(stream)
+					: new InputStreamReader(stream, encoding);
+		char[] contents;
+		if (length == -1) {
+			contents = CharOperation.NO_CHAR;
+			int contentsLength = 0;
+			int amountRead = -1;
+			do {
+				int amountRequested = Math.max(stream.available(), DEFAULT_READING_SIZE);  // read at least 8K
+
+				// resize contents if needed
+				if (contentsLength + amountRequested > contents.length) {
+					System.arraycopy(
+						contents,
+						0,
+						contents = new char[contentsLength + amountRequested],
+						0,
+						contentsLength);
+				}
+
+				// read as many chars as possible
+				amountRead = reader.read(contents, contentsLength, amountRequested);
+
+				if (amountRead > 0) {
+					// remember length of contents
+					contentsLength += amountRead;
+				}
+			} while (amountRead != -1);
+
+			// resize contents if necessary
+			if (contentsLength < contents.length) {
+				System.arraycopy(
+					contents,
+					0,
+					contents = new char[contentsLength],
+					0,
+					contentsLength);
+			}
+		} else {
+			contents = new char[length];
+			int len = 0;
+			int readSize = 0;
+			while ((readSize != -1) && (len != length)) {
+				// See PR 1FMS89U
+				// We record first the read size. In this case len is the actual read size.
+				len += readSize;
+				readSize = reader.read(contents, len, length - len);
+			}
+			// See PR 1FMS89U
+			// Now we need to resize in case the default encoding used more than one byte for each
+			// character
+			if (len != length)
+				System.arraycopy(contents, 0, (contents = new char[len]), 0, len);
+		}
+
+		return contents;
+	}
+	
+
+	/**
+	 * Helper method - returns the targeted item (IResource if internal or java.io.File if external), 
+	 * or null if unbound
+	 * Internal items must be referred to using container relative paths.
+	 */
+	public static Object getTarget(IContainer container, IPath path, boolean checkResourceExistence) {
+
+		if (path == null) return null;
+	
+		// lookup - inside the container
+		if (path.getDevice() == null) { // container relative paths should not contain a device 
+													// (see http://dev.eclipse.org/bugs/show_bug.cgi?id=18684)
+													// (case of a workspace rooted at d:\ )
+			IResource resource = container.findMember(path);
+			if (resource != null){
+				if (!checkResourceExistence ||resource.exists()) return resource;
+				return null;
+			}
+		}
+	
+		// if path is relative, it cannot be an external path
+		// (see http://dev.eclipse.org/bugs/show_bug.cgi?id=22517)
+		if (!path.isAbsolute()) return null; 
+
+		// lookup - outside the container
+		File externalFile = new File(path.toOSString());
+		if (!checkResourceExistence) {
+			return externalFile;
+		} else if (existingExternalFiles.contains(externalFile)) {
+			return externalFile;
+		} else {
+			//TODO: BOG do we need to add something here?
+			/*
+			if (JavaModelManager.ZIP_ACCESS_VERBOSE) {
+				System.out.println("(" + Thread.currentThread() + ") [JavaModel.getTarget(...)] Checking existence of " + path.toString()); //$NON-NLS-1$ //$NON-NLS-2$
+			}
+			*/
+			if (externalFile.exists()) {
+				// cache external file
+				existingExternalFiles.add(externalFile);
+				return externalFile;
+			}
+		}
+		
+		return null;	
+	}
+		/**
+		 * A set of java.io.Files used as a cache of external jars that 
+		 * are known to be existing.
+		 * Note this cache is kept for the whole session.
+		 */ 
+		public static HashSet existingExternalFiles = new HashSet();
+		
+	/*
+	 * Returns whether the given resource matches one of the exclusion patterns.
+	 * 
+	 * @see IClasspathEntry#getExclusionPatterns
+	 */
+	public final static boolean isExcluded(IResource resource, char[][] exclusionPatterns) {
+		IPath path = resource.getFullPath();
+		// ensure that folders are only excluded if all of their children are excluded
+		if (resource.getType() == IResource.FOLDER)
+			path = path.append("*"); //$NON-NLS-1$
+		return isExcluded(path, exclusionPatterns);
+	}
+	
+	/*
+	 * Returns whether the given resource path matches one of the exclusion
+	 * patterns.
+	 * 
+	 * @see IClasspathEntry#getExclusionPatterns
+	 */
+	public final static boolean isExcluded(IPath resourcePath, char[][] exclusionPatterns) {
+		if (exclusionPatterns == null) return false;
+		char[] path = resourcePath.toString().toCharArray();
+		for (int i = 0, length = exclusionPatterns.length; i < length; i++)
+			if (CharOperation.pathMatch(exclusionPatterns[i], path, true, '/'))
+				return true;
+		return false;
+	}	
+		
+}
+
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/AbstractIndexer.java

@@ -0,0 +1,143 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 31, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+import java.io.IOException;
+
+import org.eclipse.cdt.core.parser.ast.ClassKind;
+import org.eclipse.cdt.core.parser.ast.IASTClassSpecifier;
+import org.eclipse.cdt.core.search.ICSearchConstants;
+import org.eclipse.cdt.internal.core.index.IDocument;
+import org.eclipse.cdt.internal.core.index.IIndexer;
+import org.eclipse.cdt.internal.core.index.IIndexerOutput;
+import org.eclipse.cdt.internal.core.search.CharOperation;
+
+/**
+ * @author bgheorgh
+ */
+
+public abstract class AbstractIndexer implements IIndexer, IIndexConstants, ICSearchConstants {
+	
+	IIndexerOutput output;
+	final static int CLASS = 1;
+	final static int STRUCT = 2;
+	final static int UNION = 3;
+	
+	public AbstractIndexer() {
+		super();
+	}
+	
+	public void addClassSpecifier(IASTClassSpecifier classSpecification){
+		if (classSpecification.getClassKind().equals(ClassKind.CLASS))
+		{
+			this.output.addRef(encodeTypeEntry(classSpecification.getName().toCharArray(),CLASS));
+		}		
+		else if (classSpecification.getClassKind().equals(ClassKind.STRUCT))
+		{
+			this.output.addRef(encodeTypeEntry(classSpecification.getName().toCharArray(),STRUCT));
+		}
+		else if (classSpecification.getClassKind().equals(ClassKind.UNION))
+		{
+			this.output.addRef(encodeTypeEntry(classSpecification.getName().toCharArray(),UNION));			
+		}
+	}
+	
+	public void addConstructorDeclaration(){
+		
+	}
+	public void addConstructorReference(){
+
+	}
+	
+	public void addMemberDeclaration(){
+	
+	}
+	public void addMemberReference(){
+	
+	}
+
+	public void addFunctionDeclaration(){
+	
+	}
+	
+	public void addFunctionReference(){
+	
+	}
+	
+	public void addNameReference(){
+		
+	}
+	
+	private void addSuperTypeReference(int modifiers, char[] packageName, char[] typeName, char[][] enclosingTypeNames, char classOrInterface, char[] superTypeName, char superClassOrInterface){
+
+	}
+	
+	public void addTypeReference(char[] typeName){
+		this.output.addRef(CharOperation.concat(TYPE_REF, CharOperation.lastSegment(typeName, '.')));
+	}
+	/**
+	 * Type entries are encoded as follow: 'typeDecl/' ('C' | 'S' | 'U' ) '/'  TypeName '/' 
+	 */
+	 protected static final char[] encodeTypeEntry( char[] typeName, int classType) {
+		
+		int pos;
+		//3 - 1 for SUFFIX letter, 2 Separators
+		char[] result = new char[TYPE_DECL_LENGTH + typeName.length + 3];
+		System.arraycopy(TYPE_DECL, 0, result, 0, pos = TYPE_DECL_LENGTH);
+		switch (classType)
+		{
+			case(CLASS):
+			result[pos++] = CLASS_SUFFIX;
+			break;
+			
+			case(STRUCT):
+			result[pos++] = STRUCT_SUFFIX;
+			break;
+			
+			case(UNION):
+			result[pos++] = UNION_SUFFIX;
+			break;
+		}
+		
+		result[pos++] = SEPARATOR;
+		System.arraycopy(typeName, 0, result, pos, typeName.length);
+		pos += typeName.length;
+		result[pos++] = SEPARATOR;
+		
+		return result;
+	}
+	/**
+	 * Returns the file types the <code>IIndexer</code> handles.
+	 */
+	public abstract String[] getFileTypes();
+	/**
+	 * @see IIndexer#index(IDocument document, IIndexerOutput output)
+	 */
+	public void index(IDocument document, IIndexerOutput output) throws IOException {
+		this.output = output;
+		if (shouldIndex(document)) indexFile(document);
+	}
+	
+	protected abstract void indexFile(IDocument document) throws IOException;
+	/**
+	 * @see IIndexer#shouldIndex(IDocument document)
+	 */
+	public boolean shouldIndex(IDocument document) {
+		String type = document.getType();
+		String[] supportedTypes = this.getFileTypes();
+		for (int i = 0; i < supportedTypes.length; ++i) {
+			if (supportedTypes[i].equals(type))
+				return true;
+		}
+		return false;
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/AddCompilationUnitToIndex.java

@@ -0,0 +1,42 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+import java.io.IOException;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.cdt.internal.core.index.IIndex;
+import org.eclipse.cdt.internal.core.index.impl.IFileDocument;
+
+public class AddCompilationUnitToIndex extends AddFileToIndex {
+	char[] contents;
+
+	public AddCompilationUnitToIndex(IFile resource, IPath indexedContainer, IndexManager manager) {
+		super(resource, indexedContainer, manager);
+	}
+	protected boolean indexDocument(IIndex index) throws IOException {
+		if (!initializeContents()) return false;
+		index.add(new IFileDocument(resource, this.contents), new SourceIndexer(resource));
+		return true;
+	}
+	public boolean initializeContents() {
+		if (this.contents == null) {
+			try {
+				IPath location = resource.getLocation();
+				if (location != null)
+					this.contents = org.eclipse.cdt.internal.core.search.Util.getFileCharContent(location.toFile(), null);
+			} catch (IOException e) {
+			}
+		}
+		return this.contents != null;
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/AddFileToIndex.java

@@ -0,0 +1,56 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+import java.io.IOException;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.cdt.internal.core.index.IIndex;
+import org.eclipse.cdt.internal.core.search.processing.JobManager;
+
+public abstract class AddFileToIndex extends IndexRequest {
+	IFile resource;
+
+	public AddFileToIndex(IFile resource, IPath indexPath, IndexManager manager) {
+		super(indexPath, manager);
+		this.resource = resource;
+	}
+	
+	public boolean execute(IProgressMonitor progressMonitor) {
+		if (progressMonitor != null && progressMonitor.isCanceled()) return true;
+		/* ensure no concurrent write access to index */
+		IIndex index = manager.getIndex(this.indexPath, true, /*reuse index file*/ true /*create if none*/);
+		if (index == null) return true;
+		ReadWriteMonitor monitor = manager.getMonitorFor(index);
+		if (monitor == null) return true; // index got deleted since acquired
+		try {
+			monitor.enterWrite(); // ask permission to write
+			if (!indexDocument(index)) return false;
+		} catch (IOException e) {
+			if (JobManager.VERBOSE) {
+				JobManager.verbose("-> failed to index " + this.resource + " because of the following exception:"); //$NON-NLS-1$ //$NON-NLS-2$
+				e.printStackTrace();
+			}
+			return false;
+		} finally {
+			monitor.exitWrite(); // free write lock
+		}
+		return true;
+	}
+	
+	protected abstract boolean indexDocument(IIndex index) throws IOException;
+	
+	public String toString() {
+		return "indexing " + this.resource.getFullPath(); //$NON-NLS-1$
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/AddFolderToIndex.java

@@ -0,0 +1,92 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IProject;
+import org.eclipse.core.resources.IResource;
+import org.eclipse.core.resources.IResourceProxy;
+import org.eclipse.core.resources.IResourceProxyVisitor;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.cdt.internal.core.search.Util;
+import org.eclipse.cdt.internal.core.index.IIndex;
+import org.eclipse.cdt.internal.core.search.processing.JobManager;
+
+class AddFolderToIndex extends IndexRequest {
+	IPath folderPath;
+	IProject project;
+	char[][] exclusionPattern;
+
+	public AddFolderToIndex(IPath folderPath, IProject project, char[][] exclusionPattern, IndexManager manager) {
+		super(project.getFullPath(), manager);
+		this.folderPath = folderPath;
+		this.project = project;
+		this.exclusionPattern = exclusionPattern;
+	}
+	
+	public boolean execute(IProgressMonitor progressMonitor) {
+		if (progressMonitor != null && progressMonitor.isCanceled()) return true;
+		if (!project.isAccessible()) return true; // nothing to do
+		IResource folder = this.project.getParent().findMember(this.folderPath);
+		if (folder == null || folder.getType() == IResource.FILE) return true; // nothing to do, source folder was removed
+
+		/* ensure no concurrent write access to index */
+		IIndex index = manager.getIndex(this.indexPath, true, /*reuse index file*/ true /*create if none*/);
+		if (index == null) return true;
+		ReadWriteMonitor monitor = manager.getMonitorFor(index);
+		if (monitor == null) return true; // index got deleted since acquired
+
+		try {
+			monitor.enterRead(); // ask permission to read
+
+			final IPath container = this.indexPath;
+			final IndexManager indexManager = this.manager;
+			final char[][] pattern = exclusionPattern;
+			folder.accept(
+				new IResourceProxyVisitor() {
+					public boolean visit(IResourceProxy proxy) throws CoreException {
+						switch(proxy.getType()) {
+							case IResource.FILE :
+//							TODO: BOG Put the file name checking back
+								//if (Util.isJavaFileName(proxy.getName())) {
+									IResource resource = proxy.requestResource();
+									if (pattern == null || !Util.isExcluded(resource, pattern))
+										indexManager.addSource((IFile)resource, container);
+								//}
+								//return false;
+							case IResource.FOLDER :
+								if (pattern != null && Util.isExcluded(proxy.requestResource(), pattern))
+									return false;
+						}
+						return true;
+					}
+				},
+				IResource.NONE
+			);
+		} catch (CoreException e) {
+			if (JobManager.VERBOSE) {
+				JobManager.verbose("-> failed to add " + this.folderPath + " to index because of the following exception:"); //$NON-NLS-1$ //$NON-NLS-2$
+				e.printStackTrace();
+			}
+			return false;
+		} finally {
+			monitor.exitRead(); // free read lock
+		}
+		return true;
+	}
+	
+	public String toString() {
+		return "adding " + this.folderPath + " to index " + this.indexPath; //$NON-NLS-1$ //$NON-NLS-2$
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/IIndexConstants.java

@@ -0,0 +1,76 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+/**
+ * @author bgheorgh
+ */
+public interface IIndexConstants {
+
+	/* index encoding */
+	char[] REF= "ref/".toCharArray(); //$NON-NLS-1$
+	
+	char[] TYPE_REF= "typeRef/".toCharArray(); //$NON-NLS-1$
+	char[] TYPE_DECL = "typeDecl/".toCharArray(); //$NON-NLS-1$
+	int TYPE_DECL_LENGTH = 9;
+	
+	char[] FUNCTION_REF= "functionRef/".toCharArray(); //$NON-NLS-1$
+	char[] FUNCTION_DECL= "functionDecl/".toCharArray(); //$NON-NLS-1$
+	
+	char[] CONSTRUCTOR_REF= "constructorRef/".toCharArray(); //$NON-NLS-1$
+	char[] CONSTRUCTOR_DECL= "constructorDecl/".toCharArray(); //$NON-NLS-1$
+	
+	char[] NAMESPACE_REF= "namespaceRef/".toCharArray(); //$NON-NLS-1$
+	char[] NAMESPACE_DECL= "namespaceDecl/".toCharArray(); //$NON-NLS-1$
+		
+	char[] MEMBER_REF= "memberRef/".toCharArray(); //$NON-NLS-1$
+	char[] MEMBER_DECL= "memberDecl/".toCharArray(); //$NON-NLS-1$
+
+	//a Var REF will be treated as  a typeREF
+	//char[] VAR_REF= "varRef/".toCharArray(); //$NON-NLS-1$
+	
+	//a Struct REF will be treated as  a typeREF
+	//char[] STRUCT_REF= "structRef/".toCharArray(); //$NON-NLS-1$
+	
+	//a Enum REF will be treated as a typeREF
+	//char[] ENUM_REF= "enumRef/".toCharArray(); //$NON-NLS-1$
+
+    //a UNION REF will be treated as a typeREF
+	//char[] UNION_REF= "unionRef/".toCharArray(); //$NON-NLS-1$
+	
+	
+	char[] SUPER_REF = "superRef/".toCharArray(); //$NON-NLS-1$
+
+	char[] CLASS_DECL= "typeDecl/C/".toCharArray(); //$NON-NLS-1$
+	char[] VAR_DECL= "typeDecl/V/".toCharArray(); //$NON-NLS-1$
+	char[] STRUCT_DECL= "typeDecl/S/".toCharArray(); //$NON-NLS-1$
+	char[] ENUM_DECL= "typeDecl/E/".toCharArray(); //$NON-NLS-1$
+	char[] UNION_DECL= "typeDecl/U/".toCharArray(); //$NON-NLS-1$
+	
+	char[] OBJECT = "Object".toCharArray(); //$NON-NLS-1$
+	char[][] COUNTS= 
+		new char[][] { new char[] {'0'}, new char[] {'1'}, new char[] {'2'}, new char[] {'3'}, new char[] {'4'}, new char[] {'5'}, new char[] {'6'}, new char[] {'7'}, new char[] {'8'}, new char[] {'9'}
+	};
+	char CLASS_SUFFIX = 'C';
+	char VAR_SUFFIX = 'V';
+	char STRUCT_SUFFIX = 'S';
+	char ENUM_SUFFIX = 'E';
+	char UNION_SUFFIX = 'U';
+	
+	char TYPE_SUFFIX = 0;
+	char SEPARATOR= '/';
+
+	char[] ONE_STAR = new char[] {'*'};
+	char[][] ONE_STAR_CHAR = new char[][] {ONE_STAR};
+
+	// used as special marker for enclosing type name of local and anonymous classes
+	char[] ONE_ZERO = new char[] {'0'}; 
+	char[][] ONE_ZERO_CHAR = new char[][] {ONE_ZERO};
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/IndexAllProject.java

@@ -0,0 +1,214 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+import java.io.IOException;
+import java.util.HashSet;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IProject;
+import org.eclipse.core.resources.IResource;
+import org.eclipse.core.resources.IResourceProxy;
+import org.eclipse.core.resources.IResourceProxyVisitor;
+import org.eclipse.core.resources.IWorkspaceRoot;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
+import  org.eclipse.cdt.core.model.ICProject;
+import org.eclipse.cdt.internal.core.search.Util;
+import org.eclipse.cdt.internal.core.index.IIndex;
+import org.eclipse.cdt.internal.core.index.IQueryResult;
+import org.eclipse.cdt.internal.core.index.impl.IFileDocument;
+import org.eclipse.cdt.internal.core.search.processing.JobManager;
+import org.eclipse.cdt.internal.core.search.SimpleLookupTable;
+import org.eclipse.cdt.internal.core.model.CModel;
+import org.eclipse.cdt.internal.core.model.CModelManager;
+import org.eclipse.cdt.core.model.ICElement;
+
+
+public class IndexAllProject extends IndexRequest {
+	IProject project;
+
+	public IndexAllProject(IProject project, IndexManager manager) {
+		super(project.getFullPath(), manager);
+		this.project = project;
+	}
+	
+	public boolean equals(Object o) {
+		if (o instanceof IndexAllProject)
+			return this.project.equals(((IndexAllProject) o).project);
+		return false;
+	}
+	/**
+	 * Ensure consistency of a project index. Need to walk all nested resources,
+	 * and discover resources which have either been changed, added or deleted
+	 * since the index was produced.
+	 */
+	public boolean execute(IProgressMonitor progressMonitor) {
+
+		if (progressMonitor != null && progressMonitor.isCanceled()) return true;
+		if (!project.isAccessible()) return true; // nothing to do
+
+		IIndex index = this.manager.getIndex(this.indexPath, true, /*reuse index file*/ true /*create if none*/);
+		if (index == null) return true;
+		ReadWriteMonitor monitor = this.manager.getMonitorFor(index);
+		if (monitor == null) return true; // index got deleted since acquired
+
+		try {
+			monitor.enterRead(); // ask permission to read
+			saveIfNecessary(index, monitor);
+
+			IQueryResult[] results = index.queryInDocumentNames(""); // all file names //$NON-NLS-1$
+			int max = results == null ? 0 : results.length;
+			final SimpleLookupTable indexedFileNames = new SimpleLookupTable(max == 0 ? 33 : max + 11);
+			final String OK = "OK"; //$NON-NLS-1$
+			final String DELETED = "DELETED"; //$NON-NLS-1$
+			for (int i = 0; i < max; i++)
+				indexedFileNames.put(results[i].getPath(), DELETED);
+			final long indexLastModified = max == 0 ? 0L : index.getIndexFile().lastModified();
+
+			CModel model = (CModel) CModelManager.getDefault().getCModel();
+			ICProject cProject = model.getCProject(project.getName());
+			ICElement[] kids = cProject.getChildren();
+			IPath cProjectPath = cProject.getPath(); 
+			
+			IWorkspaceRoot root = this.project.getWorkspace().getRoot();
+			IResource sourceFolder = root.findMember(cProjectPath);
+			
+			if (this.isCancelled) return false;
+
+			if (sourceFolder != null) {
+						
+				// collect output locations if source is project (see http://bugs.eclipse.org/bugs/show_bug.cgi?id=32041)
+				final HashSet outputs = new HashSet();
+			
+				final boolean hasOutputs = !outputs.isEmpty();
+						
+				final char[][] patterns = null;
+				if (max == 0) {
+					sourceFolder.accept(
+							new IResourceProxyVisitor() {
+								public boolean visit(IResourceProxy proxy) {
+									if (isCancelled) return false;
+									switch(proxy.getType()) {
+										case IResource.FILE :
+										//TODO: BOG Put the file name checking back
+										//if (Util.isCCFileName(proxy.getName())) {
+											IResource resource = proxy.requestResource();
+											if (resource.getLocation() != null && (patterns == null || !Util.isExcluded(resource, patterns))) {
+												String name = new IFileDocument((IFile) resource).getName();
+												indexedFileNames.put(name, resource);
+											}
+										//}
+										//	return false;
+			
+										case IResource.FOLDER :
+												if (patterns != null && Util.isExcluded(proxy.requestResource(), patterns))
+													return false;
+												if (hasOutputs && outputs.contains(proxy.requestFullPath())) {
+													return false;
+												}
+										}
+										return true;
+								 }
+								},
+								IResource.NONE
+							);
+						} else {
+							sourceFolder.accept(
+								new IResourceProxyVisitor() {
+									public boolean visit(IResourceProxy proxy) {
+										if (isCancelled) return false;
+										switch(proxy.getType()) {
+											case IResource.FILE :
+//											TODO: BOG Put the file name checking back
+												//if (Util.isCCFileName(proxy.getName())) {
+													IResource resource = proxy.requestResource();
+													IPath path = resource.getLocation();
+													if (path != null && (patterns == null || !Util.isExcluded(resource, patterns))) {
+														String name = new IFileDocument((IFile) resource).getName();
+														indexedFileNames.put(name,
+															indexedFileNames.get(name) == null || indexLastModified < path.toFile().lastModified()
+																? (Object) resource
+																: (Object) OK);
+													}
+												//}
+												//return false;
+											case IResource.FOLDER :
+												if (patterns != null && Util.isExcluded(proxy.requestResource(), patterns))
+													return false;
+												if (hasOutputs && outputs.contains(proxy.requestFullPath())) {
+													return false;
+												}
+										}
+										return true;
+									}
+								},
+								IResource.NONE
+							);
+						}
+					}
+					
+			Object[] names = indexedFileNames.keyTable;
+			Object[] values = indexedFileNames.valueTable;
+			boolean shouldSave = false;
+			for (int i = 0, length = names.length; i < length; i++) {
+				String name = (String) names[i];
+				if (name != null) {
+					if (this.isCancelled) return false;
+
+					Object value = values[i];
+					if (value != OK) {
+						shouldSave = true;
+						if (value == DELETED)
+							this.manager.remove(name, this.indexPath);
+						else
+							this.manager.addSource((IFile) value, this.indexPath);
+					}
+				}
+			}
+
+			// request to save index when all cus have been indexed
+			if (shouldSave)
+				this.manager.request(new SaveIndex(this.indexPath, this.manager));
+
+		} catch (CoreException e) {
+			if (JobManager.VERBOSE) {
+				JobManager.verbose("-> failed to index " + this.project + " because of the following exception:"); //$NON-NLS-1$ //$NON-NLS-2$
+				e.printStackTrace();
+			}
+			this.manager.removeIndex(this.indexPath);
+			return false;
+		} catch (IOException e) {
+			if (JobManager.VERBOSE) {
+				JobManager.verbose("-> failed to index " + this.project + " because of the following exception:"); //$NON-NLS-1$ //$NON-NLS-2$
+				e.printStackTrace();
+			}
+			this.manager.removeIndex(this.indexPath);
+			return false;
+		} finally {
+			monitor.exitRead(); // free read lock
+		}
+		return true;
+	}
+	
+	public int hashCode() {
+		return this.project.hashCode();
+	}
+	
+	protected Integer updatedIndexState() {
+		return IndexManager.REBUILDING_STATE;
+	}
+	
+	public String toString() {
+		return "indexing project " + this.project.getFullPath(); //$NON-NLS-1$
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/IndexManager.java

@@ -0,0 +1,611 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+ 
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.zip.CRC32;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IProject;
+import org.eclipse.core.resources.IWorkspace;
+import org.eclipse.core.resources.ResourcesPlugin;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.QualifiedName;
+import org.eclipse.cdt.internal.core.search.processing.JobManager;
+import org.eclipse.cdt.internal.core.search.processing.IJob;
+import org.eclipse.cdt.internal.core.search.SimpleLookupTable;
+import org.eclipse.cdt.internal.core.search.CharOperation;
+import org.eclipse.cdt.internal.core.index.IIndex;
+import org.eclipse.cdt.internal.core.index.impl.Index;
+import org.eclipse.cdt.internal.core.model.CProject;
+import org.eclipse.cdt.core.CCorePlugin;
+
+public class IndexManager extends JobManager implements IIndexConstants {
+	/* number of file contents in memory */
+	public static int MAX_FILES_IN_MEMORY = 0;
+
+	public IWorkspace workspace;
+	public SimpleLookupTable indexNames = new SimpleLookupTable();
+	private Map indexes = new HashMap(5);
+
+	/* read write monitors */
+	private Map monitors = new HashMap(5);
+
+	/* need to save ? */
+	private boolean needToSave = false;
+	private static final CRC32 checksumCalculator = new CRC32();
+	private IPath javaPluginLocation = null;
+
+	/* can only replace a current state if its less than the new one */
+	private SimpleLookupTable indexStates = null;
+	private File savedIndexNamesFile =
+		new File(getJavaPluginWorkingLocation().append("savedIndexNames.txt").toOSString()); //$NON-NLS-1$
+	public static Integer SAVED_STATE = new Integer(0);
+	public static Integer UPDATING_STATE = new Integer(1);
+	public static Integer UNKNOWN_STATE = new Integer(2);
+	public static Integer REBUILDING_STATE = new Integer(3);
+
+	public synchronized void aboutToUpdateIndex(IPath path, Integer newIndexState) {
+		// newIndexState is either UPDATING_STATE or REBUILDING_STATE
+		// must tag the index as inconsistent, in case we exit before the update job is started
+		String indexName = computeIndexName(path);
+		Object state = getIndexStates().get(indexName);
+		Integer currentIndexState = state == null ? UNKNOWN_STATE : (Integer) state;
+		if (currentIndexState.equals(REBUILDING_STATE)) return; // already rebuilding the index
+
+		int compare = newIndexState.compareTo(currentIndexState);
+		if (compare > 0) {
+			// so UPDATING_STATE replaces SAVED_STATE and REBUILDING_STATE replaces everything
+			updateIndexState(indexName, newIndexState);
+		} else if (compare < 0 && this.indexes.get(path) == null) {
+			// if already cached index then there is nothing more to do
+			rebuildIndex(indexName, path);
+		}
+	}
+	/**
+	 * Not at the moment...
+	 * @param resource
+	 * @param indexedContainer
+	 */
+	/* 
+	public void addBinary(IFile resource, IPath indexedContainer){
+		if (JavaCore.getPlugin() == null) return;	
+		AddClassFileToIndex job = new AddClassFileToIndex(resource, indexedContainer, this);
+		if (this.awaitingJobsCount() < MAX_FILES_IN_MEMORY) {
+			// reduces the chance that the file is open later on, preventing it from being deleted
+			if (!job.initializeContents()) return;
+		}
+		request(job);
+	}
+	*/
+	/**
+	 * Trigger addition of a resource to an index
+	 * Note: the actual operation is performed in background
+	 */
+	public void addSource(IFile resource, IPath indexedContainer){
+		if (CCorePlugin.getDefault() == null) return;	
+		AddCompilationUnitToIndex job = new AddCompilationUnitToIndex(resource, indexedContainer, this);
+		if (this.awaitingJobsCount() < MAX_FILES_IN_MEMORY) {
+			// reduces the chance that the file is open later on, preventing it from being deleted
+			if (!job.initializeContents()) return;
+		}
+		request(job);
+	}
+	
+	String computeIndexName(IPath path) {
+		String name = (String) indexNames.get(path);
+		if (name == null) {
+			String pathString = path.toOSString();
+			checksumCalculator.reset();
+			checksumCalculator.update(pathString.getBytes());
+			String fileName = Long.toString(checksumCalculator.getValue()) + ".index"; //$NON-NLS-1$
+			if (VERBOSE)
+				JobManager.verbose("-> index name for " + pathString + " is " + fileName); //$NON-NLS-1$ //$NON-NLS-2$
+			name = getJavaPluginWorkingLocation().append(fileName).toOSString();
+			indexNames.put(path, name);
+		}
+		return name;
+	}
+	/**
+	 * Returns the index for a given project, according to the following algorithm:
+	 * - if index is already in memory: answers this one back
+	 * - if (reuseExistingFile) then read it and return this index and record it in memory
+	 * - if (createIfMissing) then create a new empty index and record it in memory
+	 * 
+	 * Warning: Does not check whether index is consistent (not being used)
+	 */
+	public synchronized IIndex getIndex(IPath path, boolean reuseExistingFile, boolean createIfMissing) {
+		// Path is already canonical per construction
+		IIndex index = (IIndex) indexes.get(path);
+		if (index == null) {
+			String indexName = computeIndexName(path);
+			Object state = getIndexStates().get(indexName);
+			Integer currentIndexState = state == null ? UNKNOWN_STATE : (Integer) state;
+			if (currentIndexState == UNKNOWN_STATE) {
+				// should only be reachable for query jobs
+				// IF you put an index in the cache, then AddJarFileToIndex fails because it thinks there is nothing to do
+				rebuildIndex(indexName, path);
+				return null;
+			}
+
+			// index isn't cached, consider reusing an existing index file
+			if (reuseExistingFile) {
+				File indexFile = new File(indexName);
+				if (indexFile.exists()) { // check before creating index so as to avoid creating a new empty index if file is missing
+					try {
+						index = new Index(indexName, "Index for " + path.toOSString(), true /*reuse index file*/); //$NON-NLS-1$
+						indexes.put(path, index);
+						monitors.put(index, new ReadWriteMonitor());
+						return index;
+					} catch (IOException e) {
+						// failed to read the existing file or its no longer compatible
+						if (currentIndexState != REBUILDING_STATE) { // rebuild index if existing file is corrupt, unless the index is already being rebuilt
+							if (VERBOSE)
+								JobManager.verbose("-> cannot reuse existing index: "+indexName+" path: "+path.toOSString()); //$NON-NLS-1$ //$NON-NLS-2$
+							rebuildIndex(indexName, path);
+							return null;
+						} else {
+							index = null; // will fall thru to createIfMissing & create a empty index for the rebuild all job to populate
+						}
+					}
+				}
+				if (currentIndexState == SAVED_STATE) { // rebuild index if existing file is missing
+					rebuildIndex(indexName, path);
+					return null;
+				}
+			} 
+			// index wasn't found on disk, consider creating an empty new one
+			if (createIfMissing) {
+				try {
+					if (VERBOSE)
+						JobManager.verbose("-> create empty index: "+indexName+" path: "+path.toOSString()); //$NON-NLS-1$ //$NON-NLS-2$
+					index = new Index(indexName, "Index for " + path.toOSString(), false /*do not reuse index file*/); //$NON-NLS-1$
+					indexes.put(path, index);
+					monitors.put(index, new ReadWriteMonitor());
+					return index;
+				} catch (IOException e) {
+					if (VERBOSE)
+						JobManager.verbose("-> unable to create empty index: "+indexName+" path: "+path.toOSString()); //$NON-NLS-1$ //$NON-NLS-2$
+					// The file could not be created. Possible reason: the project has been deleted.
+					return null;
+				}
+			}
+		}
+		//System.out.println(" index name: " + path.toOSString() + " <----> " + index.getIndexFile().getName());	
+		return index;
+	}
+	
+	private SimpleLookupTable getIndexStates() {
+		if (indexStates != null) return indexStates;
+
+		this.indexStates = new SimpleLookupTable();
+		char[] savedIndexNames = readIndexState();
+		if (savedIndexNames.length > 0) {
+			char[][] names = CharOperation.splitOn('\n', savedIndexNames);
+			for (int i = 0, l = names.length; i < l; i++) {
+				char[] name = names[i];
+				if (name.length > 0)
+					this.indexStates.put(new String(name), SAVED_STATE);
+			}
+		}
+		return this.indexStates;
+	}
+	
+	private IPath getJavaPluginWorkingLocation() {
+		if (this.javaPluginLocation != null) return this.javaPluginLocation;
+
+		return this.javaPluginLocation = CCorePlugin.getDefault().getStateLocation();
+	}
+	/**
+	 * Index access is controlled through a read-write monitor so as
+	 * to ensure there is no concurrent read and write operations
+	 * (only concurrent reading is allowed).
+	 */
+	public ReadWriteMonitor getMonitorFor(IIndex index){
+		return (ReadWriteMonitor) monitors.get(index);
+	}
+	/**
+	 * Trigger addition of the entire content of a project
+	 * Note: the actual operation is performed in background 
+	 */
+	public void indexAll(IProject project) {
+		if (CCorePlugin.getDefault() == null) return;
+		
+		 /******
+		 *TODO: Remove these methods once the new indexer is
+		 *fully integrated
+		 */
+		 if (!isEnabled(project)) return;
+
+		// check if the same request is not already in the queue
+		IndexRequest request = new IndexAllProject(project, this);
+		for (int i = this.jobEnd; i > this.jobStart; i--) // NB: don't check job at jobStart, as it may have already started (see http://bugs.eclipse.org/bugs/show_bug.cgi?id=32488)
+			if (request.equals(this.awaitingJobs[i])) return;
+		this.request(request);
+	}
+	/**
+	 * Index the content of the given source folder.
+	 */
+	public void indexSourceFolder(CProject javaProject, IPath sourceFolder, final char[][] exclusionPattern) {
+		IProject project = javaProject.getProject();
+		if (this.jobEnd > this.jobStart) {
+			// check if a job to index the project is not already in the queue
+			IndexRequest request = new IndexAllProject(project, this);
+			for (int i = this.jobEnd; i > this.jobStart; i--) // NB: don't check job at jobStart, as it may have already started (see http://bugs.eclipse.org/bugs/show_bug.cgi?id=32488)
+				if (request.equals(this.awaitingJobs[i])) return;
+		}
+		this.request(new AddFolderToIndex(sourceFolder, project, exclusionPattern, this));
+	}
+	
+	public void jobWasCancelled(IPath path) {
+		Object o = this.indexes.get(path);
+		if (o instanceof IIndex) {
+			this.monitors.remove(o);
+			this.indexes.remove(path);
+		}
+		updateIndexState(computeIndexName(path), UNKNOWN_STATE);
+	}
+	/**
+	 * Advance to the next available job, once the current one has been completed.
+	 * Note: clients awaiting until the job count is zero are still waiting at this point.
+	 */
+	protected synchronized void moveToNextJob() {
+		// remember that one job was executed, and we will need to save indexes at some point
+		needToSave = true;
+		super.moveToNextJob();
+	}
+	/**
+	 * No more job awaiting.
+	 */
+	protected void notifyIdle(long idlingTime){
+		if (idlingTime > 1000 && needToSave) saveIndexes();
+	}
+	/*
+	 * For debug purpose
+	 */
+	public IIndex peekAtIndex(IPath path) {
+		return (IIndex) indexes.get(path);
+	}
+	/**
+	 * Name of the background process
+	 */
+	public String processName(){
+		return org.eclipse.cdt.internal.core.search.Util.bind("process.name"); //$NON-NLS-1$
+	}
+	
+	private void rebuildIndex(String indexName, IPath path) {
+		Object target = org.eclipse.cdt.internal.core.search.Util.getTarget(ResourcesPlugin.getWorkspace().getRoot(), path, true);
+		if (target == null) return;
+
+		if (VERBOSE)
+			JobManager.verbose("-> request to rebuild index: "+indexName+" path: "+path.toOSString()); //$NON-NLS-1$ //$NON-NLS-2$
+
+		updateIndexState(indexName, REBUILDING_STATE);
+		IndexRequest request = null;
+		if (target instanceof IProject) {
+			IProject p = (IProject) target;
+			//if (JavaProject.hasJavaNature(p))
+				request = new IndexAllProject(p, this);
+		}
+	
+		if (request != null)
+			request(request);
+	}
+	/**
+	 * Recreates the index for a given path, keeping the same read-write monitor.
+	 * Returns the new empty index or null if it didn't exist before.
+	 * Warning: Does not check whether index is consistent (not being used)
+	 */
+	public synchronized IIndex recreateIndex(IPath path) {
+		// only called to over write an existing cached index...
+		try {
+			IIndex index = (IIndex) this.indexes.get(path);
+			ReadWriteMonitor monitor = (ReadWriteMonitor) this.monitors.remove(index);
+
+			// Path is already canonical
+			String indexPath = computeIndexName(path);
+			if (VERBOSE)
+				JobManager.verbose("-> recreating index: "+indexPath+" for path: "+path.toOSString()); //$NON-NLS-1$ //$NON-NLS-2$
+			index = new Index(indexPath, "Index for " + path.toOSString(), false /*reuse index file*/); //$NON-NLS-1$
+			indexes.put(path, index);
+			monitors.put(index, monitor);
+			return index;
+		} catch (IOException e) {
+			// The file could not be created. Possible reason: the project has been deleted.
+			if (VERBOSE) {
+				JobManager.verbose("-> failed to recreate index for path: "+path.toOSString()); //$NON-NLS-1$ //$NON-NLS-2$
+				e.printStackTrace();
+			}
+			return null;
+		}
+	}
+	/**
+	 * Trigger removal of a resource to an index
+	 * Note: the actual operation is performed in background
+	 */
+	public void remove(String resourceName, IPath indexedContainer){
+		request(new RemoveFromIndex(resourceName, indexedContainer, this));
+	}
+	/**
+	 * Removes the index for a given path. 
+	 * This is a no-op if the index did not exist.
+	 */
+	public synchronized void removeIndex(IPath path) {
+		if (VERBOSE)
+			JobManager.verbose("removing index " + path); //$NON-NLS-1$
+		String indexName = computeIndexName(path);
+		File indexFile = new File(indexName);
+		if (indexFile.exists())
+			indexFile.delete();
+		Object o = this.indexes.get(path);
+		if (o instanceof IIndex)
+			this.monitors.remove(o);
+		this.indexes.remove(path);
+		updateIndexState(indexName, null);
+	}
+	/**
+	 * Removes all indexes whose paths start with (or are equal to) the given path. 
+	 */
+	public synchronized void removeIndexFamily(IPath path) {
+		// only finds cached index files... shutdown removes all non-cached index files
+		ArrayList toRemove = null;
+		Iterator iterator = this.indexes.keySet().iterator();
+		while (iterator.hasNext()) {
+			IPath indexPath = (IPath) iterator.next();
+			if (path.isPrefixOf(indexPath)) {
+				if (toRemove == null)
+					toRemove = new ArrayList();
+				toRemove.add(indexPath);
+			}
+		}
+		if (toRemove != null)
+			for (int i = 0, length = toRemove.size(); i < length; i++)
+				this.removeIndex((IPath) toRemove.get(i));
+	}
+	/**
+	 * Remove the content of the given source folder from the index.
+	 */
+	public void removeSourceFolderFromIndex(CProject javaProject, IPath sourceFolder, char[][] exclusionPatterns) {
+		IProject project = javaProject.getProject();
+		if (this.jobEnd > this.jobStart) {
+			// check if a job to index the project is not already in the queue
+			IndexRequest request = new IndexAllProject(project, this);
+			for (int i = this.jobEnd; i > this.jobStart; i--) // NB: don't check job at jobStart, as it may have already started (see http://bugs.eclipse.org/bugs/show_bug.cgi?id=32488)
+				if (request.equals(this.awaitingJobs[i])) return;
+		}
+
+		this.request(new RemoveFolderFromIndex(sourceFolder, exclusionPatterns, project, this));
+	}
+	/**
+	 * Flush current state
+	 */
+	public void reset() {
+		super.reset();
+		if (this.indexes != null) {
+			this.indexes = new HashMap(5);
+			this.monitors = new HashMap(5);
+			this.indexStates = null;
+		}
+		this.indexNames = new SimpleLookupTable();
+		this.javaPluginLocation = null;
+	}
+	
+	public void saveIndex(IIndex index) throws IOException {
+		// must have permission to write from the write monitor
+		if (index.hasChanged()) {
+			if (VERBOSE)
+				JobManager.verbose("-> saving index " + index.getIndexFile()); //$NON-NLS-1$
+			index.save();
+		}
+		String indexName = index.getIndexFile().getPath();
+		if (this.jobEnd > this.jobStart) {
+			Object indexPath = indexNames.keyForValue(indexName);
+			if (indexPath != null) {
+				for (int i = this.jobEnd; i > this.jobStart; i--) { // skip the current job
+					IJob job = this.awaitingJobs[i];
+					if (job instanceof IndexRequest)
+						if (((IndexRequest) job).indexPath.equals(indexPath)) return;
+				}
+			}
+		}
+		updateIndexState(indexName, SAVED_STATE);
+	}
+	/**
+	 * Commit all index memory changes to disk
+	 */
+	public void saveIndexes() {
+		// only save cached indexes... the rest were not modified
+		ArrayList toSave = new ArrayList();
+		synchronized(this) {
+			for (Iterator iter = this.indexes.values().iterator(); iter.hasNext();) {
+				Object o = iter.next();
+				if (o instanceof IIndex)
+					toSave.add(o);
+			}
+		}
+
+		for (int i = 0, length = toSave.size(); i < length; i++) {
+			IIndex index = (IIndex) toSave.get(i);
+			ReadWriteMonitor monitor = getMonitorFor(index);
+			if (monitor == null) continue; // index got deleted since acquired
+			try {
+				monitor.enterWrite();
+				try {
+					saveIndex(index);
+				} catch(IOException e){
+					if (VERBOSE) {
+						JobManager.verbose("-> got the following exception while saving:"); //$NON-NLS-1$
+						e.printStackTrace();
+					}
+					//Util.log(e);
+				}
+			} finally {
+				monitor.exitWrite();
+			}
+		}
+		needToSave = false;
+	}
+	
+	public void shutdown() {
+		if (VERBOSE)
+			JobManager.verbose("Shutdown"); //$NON-NLS-1$
+//TODO: BOG Put in Shutdown
+/*
+		IndexSelector indexSelector = new IndexSelector(new JavaWorkspaceScope(), null, false, this);
+		IIndex[] selectedIndexes = indexSelector.getIndexes();
+		SimpleLookupTable knownPaths = new SimpleLookupTable();
+		for (int i = 0, max = selectedIndexes.length; i < max; i++) {
+			String path = selectedIndexes[i].getIndexFile().getAbsolutePath();
+			knownPaths.put(path, path);
+		}
+
+		if (indexStates != null) {
+			Object[] indexNames = indexStates.keyTable;
+			for (int i = 0, l = indexNames.length; i < l; i++) {
+				String key = (String) indexNames[i];
+				if (key != null && !knownPaths.containsKey(key))
+					updateIndexState(key, null);
+			}
+		}
+
+		File indexesDirectory = new File(getJavaPluginWorkingLocation().toOSString());
+		if (indexesDirectory.isDirectory()) {
+			File[] indexesFiles = indexesDirectory.listFiles();
+			if (indexesFiles != null) {
+				for (int i = 0, indexesFilesLength = indexesFiles.length; i < indexesFilesLength; i++) {
+					String fileName = indexesFiles[i].getAbsolutePath();
+					if (!knownPaths.containsKey(fileName) && fileName.toLowerCase().endsWith(".index")) { //$NON-NLS-1$
+						if (VERBOSE)
+							JobManager.verbose("Deleting index file " + indexesFiles[i]); //$NON-NLS-1$
+						indexesFiles[i].delete();
+					}
+				}
+			}
+		}
+*/
+		super.shutdown();
+	}
+
+	public String toString() {
+		StringBuffer buffer = new StringBuffer(10);
+		buffer.append(super.toString());
+		buffer.append("In-memory indexes:\n"); //$NON-NLS-1$
+		int count = 0;
+		for (Iterator iter = this.indexes.values().iterator(); iter.hasNext();) {
+			buffer.append(++count).append(" - ").append(iter.next().toString()).append('\n'); //$NON-NLS-1$
+		}
+		return buffer.toString();
+	}
+
+	private char[] readIndexState() {
+		try {
+			return org.eclipse.cdt.internal.core.search.Util.getFileCharContent(savedIndexNamesFile, null);
+		} catch (IOException ignored) {
+			if (VERBOSE)
+				JobManager.verbose("Failed to read saved index file names"); //$NON-NLS-1$
+			return new char[0];
+		}
+	}
+	
+	private void updateIndexState(String indexName, Integer indexState) {
+		getIndexStates(); // ensure the states are initialized
+		if (indexState != null) {
+			if (indexState.equals(indexStates.get(indexName))) return; // not changed
+			indexStates.put(indexName, indexState);
+		} else {
+			if (!indexStates.containsKey(indexName)) return; // did not exist anyway
+			indexStates.removeKey(indexName);
+		}
+
+		BufferedWriter writer = null;
+		try {
+			writer = new BufferedWriter(new FileWriter(savedIndexNamesFile));
+			Object[] indexNames = indexStates.keyTable;
+			Object[] states = indexStates.valueTable;
+			for (int i = 0, l = states.length; i < l; i++) {
+				if (states[i] == SAVED_STATE) {
+					writer.write((String) indexNames[i]);
+					writer.write('\n');
+				}
+			}
+		} catch (IOException ignored) {
+			if (VERBOSE)
+				JobManager.verbose("Failed to write saved index file names"); //$NON-NLS-1$
+		} finally {
+			if (writer != null) {
+				try {
+					writer.close();
+				} catch (IOException e) {}
+			}
+		}
+		if (VERBOSE) {
+			String state = "?"; //$NON-NLS-1$
+			if (indexState == SAVED_STATE) state = "SAVED"; //$NON-NLS-1$
+			else if (indexState == UPDATING_STATE) state = "UPDATING"; //$NON-NLS-1$
+			else if (indexState == UNKNOWN_STATE) state = "UNKNOWN"; //$NON-NLS-1$
+			else if (indexState == REBUILDING_STATE) state = "REBUILDING"; //$NON-NLS-1$
+			JobManager.verbose("-> index state updated to: " + state + " for: "+indexName); //$NON-NLS-1$ //$NON-NLS-2$
+		}
+	}
+
+	/*************
+	 *TODO: Remove these methods once the new indexer is
+	 *fully integrated
+	 * START OF TEMP INDEXER ENABLE SECTION
+	 */
+	final static String INDEX_MODEL_ID = CCorePlugin.PLUGIN_ID + ".newindexmodel";
+	final static String ACTIVATION = "enable";
+	
+	static QualifiedName activationKey = new QualifiedName(INDEX_MODEL_ID, ACTIVATION);
+	
+	public boolean isEnabled(IProject project) {
+		String prop = null;
+		try {
+			if (project != null) {
+				prop = project.getPersistentProperty(activationKey);
+			}
+		} catch (CoreException e) {
+		}
+		return ((prop != null) && prop.equalsIgnoreCase("true"));
+	}
+	
+	public void setEnabled(IProject project, boolean on) {
+		try {
+			if (project != null) {
+				Boolean newValue = new Boolean(on);
+				Boolean oldValue = new Boolean(isEnabled(project));
+				if (!oldValue.equals(newValue)) {
+					project.setPersistentProperty(activationKey, newValue.toString());
+					if (on) {
+						indexAll(project);
+					} else {
+						//remove(project);
+					}
+				}
+			}
+		} catch (CoreException e) {
+		}
+	}
+	
+	/************
+	 * END OF TEMP INDEXER ENABLE SECTION
+	 */
+		
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/IndexRequest.java

@@ -0,0 +1,62 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+import java.io.IOException;
+
+import org.eclipse.cdt.internal.core.search.processing.IJob;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.cdt.internal.core.index.IIndex;
+
+public abstract class IndexRequest implements IJob {
+	protected boolean isCancelled = false;
+	protected IPath indexPath;
+	protected IndexManager manager;
+
+	public IndexRequest(IPath indexPath, IndexManager manager) {
+		this.indexPath = indexPath;
+		this.manager = manager;
+	}
+	
+	public boolean belongsTo(String projectName) {
+		return projectName.equals(this.indexPath.segment(0));
+	}
+	
+	public void cancel() {
+		this.manager.jobWasCancelled(this.indexPath);
+		this.isCancelled = true;
+	}
+	
+	public boolean isReadyToRun() {
+		// tag the index as inconsistent
+		this.manager.aboutToUpdateIndex(indexPath, updatedIndexState());
+		return true;
+	}
+	/*
+	 * This code is assumed to be invoked while monitor has read lock
+	 */
+	protected void saveIfNecessary(IIndex index, ReadWriteMonitor monitor) throws IOException {
+		/* if index has changed, commit these before querying */
+		if (index.hasChanged()) {
+			try {
+				monitor.exitRead(); // free read lock
+				monitor.enterWrite(); // ask permission to write
+				this.manager.saveIndex(index);
+			} finally {
+				monitor.exitWriteEnterRead(); // finished writing and reacquire read permission
+			}
+		}
+	}
+	
+	protected Integer updatedIndexState() {
+		return IndexManager.UPDATING_STATE;
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/ReadWriteMonitor.java

@@ -0,0 +1,84 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on May 30, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+/**
+ * Monitor ensuring no more than one writer working concurrently.
+ * Multiple readers are allowed to perform simultaneously.
+ */
+public class ReadWriteMonitor {
+
+	/**
+	 * <0 : writing (cannot go beyond -1, i.e one concurrent writer)
+	 * =0 : idle
+	 * >0 : reading (number of concurrent readers)
+	 */
+	private int status = 0;
+	/**
+	 * Concurrent reading is allowed
+	 * Blocking only when already writing.
+	 */
+	public synchronized void enterRead() {
+	
+		while (status < 0){
+			try {
+				wait();
+			} catch(InterruptedException e){
+			}
+		}
+		status++;
+	}
+	/**
+	 * Only one writer at a time is allowed to perform
+	 * Blocking only when already writing or reading.
+	 */
+	public synchronized void enterWrite() {
+	
+		while (status != 0){
+			try {
+				wait();
+			} catch(InterruptedException e){
+			}
+		}
+		status--;
+	}
+	/**
+	 * Only notify waiting writer(s) if last reader
+	 */
+	public synchronized void exitRead() {
+		if (--status == 0) notifyAll();
+	}
+	/**
+	 * When writing is over, all readers and possible
+	 * writers are granted permission to restart concurrently
+	 */
+	public synchronized void exitWrite() {
+		if (++status == 0) notifyAll();
+	}
+	/**
+	 * Atomic exitWrite/enterRead: Allows to keep monitor in between
+	 * exit write and next enter read.
+	 * When writing is over, all readers are granted permissing to restart
+	 * concurrently.
+	 * This is the same as:
+	 * <pre>
+	 * synchronized(monitor) {
+	 *   monitor.exitWrite();
+	 *   monitor.enterRead();
+	 * }
+	 * </pre>
+	 */
+	public synchronized void exitWriteEnterRead() {
+		this.exitWrite();
+		this.enterRead();
+	} 
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/RemoveFolderFromIndex.java

@@ -0,0 +1,71 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+import java.io.IOException;
+
+import org.eclipse.core.resources.IProject;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.Path;
+import org.eclipse.cdt.internal.core.search.Util;
+import org.eclipse.cdt.internal.core.index.IIndex;
+import org.eclipse.cdt.internal.core.index.IQueryResult;
+import org.eclipse.cdt.internal.core.search.processing.JobManager;
+
+class RemoveFolderFromIndex extends IndexRequest {
+	IPath folderPath;
+	char[][] exclusionPatterns;
+	IProject project;
+
+	public RemoveFolderFromIndex(IPath folderPath, char[][] exclusionPatterns, IProject project, IndexManager manager) {
+		super(project.getFullPath(), manager);
+		this.folderPath = folderPath;
+		this.exclusionPatterns = exclusionPatterns;
+		this.project = project;
+	}
+	
+	public boolean execute(IProgressMonitor progressMonitor) {
+		if (progressMonitor != null && progressMonitor.isCanceled()) return true;
+
+		/* ensure no concurrent write access to index */
+		IIndex index = manager.getIndex(this.indexPath, true, /*reuse index file*/ false /*create if none*/);
+		if (index == null) return true;
+		ReadWriteMonitor monitor = manager.getMonitorFor(index);
+		if (monitor == null) return true; // index got deleted since acquired
+
+		try {
+			monitor.enterRead(); // ask permission to read
+			IQueryResult[] results = index.queryInDocumentNames(this.folderPath.toString());
+			// all file names belonging to the folder or its subfolders and that are not excluded (see http://bugs.eclipse.org/bugs/show_bug.cgi?id=32607)
+			for (int i = 0, max = results == null ? 0 : results.length; i < max; i++) {
+				String documentPath = results[i].getPath();
+				if (this.exclusionPatterns == null || !Util.isExcluded(new Path(documentPath), this.exclusionPatterns)) {
+					manager.remove(documentPath, this.indexPath); // write lock will be acquired by the remove operation
+				}
+			}
+		} catch (IOException e) {
+			if (JobManager.VERBOSE) {
+				JobManager.verbose("-> failed to remove " + this.folderPath + " from index because of the following exception:"); //$NON-NLS-1$ //$NON-NLS-2$
+				e.printStackTrace();
+			}
+			return false;
+		} finally {
+			monitor.exitRead(); // free read lock
+		}
+		return true;
+	}
+	
+	public String toString() {
+		return "removing " + this.folderPath + " from index " + this.indexPath; //$NON-NLS-1$ //$NON-NLS-2$
+	}
+}
+

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/RemoveFromIndex.java

@@ -0,0 +1,56 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+import java.io.IOException;
+
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.cdt.internal.core.index.IIndex;
+import org.eclipse.cdt.internal.core.search.processing.JobManager;
+
+class RemoveFromIndex extends IndexRequest {
+	String resourceName;
+
+	public RemoveFromIndex(String resourceName, IPath indexPath, IndexManager manager) {
+		super(indexPath, manager);
+		this.resourceName = resourceName;
+	}
+	
+	public boolean execute(IProgressMonitor progressMonitor) {
+
+		if (progressMonitor != null && progressMonitor.isCanceled()) return true;
+
+		/* ensure no concurrent write access to index */
+		IIndex index = manager.getIndex(this.indexPath, true, /*reuse index file*/ false /*create if none*/);
+		if (index == null) return true;
+		ReadWriteMonitor monitor = manager.getMonitorFor(index);
+		if (monitor == null) return true; // index got deleted since acquired
+
+		try {
+			monitor.enterWrite(); // ask permission to write
+			index.remove(resourceName);
+		} catch (IOException e) {
+			if (JobManager.VERBOSE) {
+				JobManager.verbose("-> failed to remove " + this.resourceName + " from index because of the following exception:"); //$NON-NLS-1$ //$NON-NLS-2$
+				e.printStackTrace();
+			}
+			return false;
+		} finally {
+			monitor.exitWrite(); // free write lock
+		}
+		return true;
+	}
+	
+	public String toString() {
+		return "removing " + this.resourceName + " from index " + this.indexPath; //$NON-NLS-1$ //$NON-NLS-2$
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/SaveIndex.java

@@ -0,0 +1,55 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+import java.io.IOException;
+
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.cdt.internal.core.index.IIndex;
+import org.eclipse.cdt.internal.core.search.processing.JobManager;
+
+/*
+ * Save the index of a project.
+ */
+public class SaveIndex extends IndexRequest {
+	public SaveIndex(IPath indexPath, IndexManager manager) {
+		super(indexPath, manager);
+	}
+	
+	public boolean execute(IProgressMonitor progressMonitor) {
+		if (progressMonitor != null && progressMonitor.isCanceled()) return true;
+
+		/* ensure no concurrent write access to index */
+		IIndex index = this.manager.getIndex(this.indexPath, true /*reuse index file*/, false /*don't create if none*/);
+		if (index == null) return true;
+		ReadWriteMonitor monitor = this.manager.getMonitorFor(index);
+		if (monitor == null) return true; // index got deleted since acquired
+
+		try {
+			monitor.enterWrite(); // ask permission to write
+			this.manager.saveIndex(index);
+		} catch (IOException e) {
+			if (JobManager.VERBOSE) {
+				JobManager.verbose("-> failed to save index " + this.indexPath + " because of the following exception:"); //$NON-NLS-1$ //$NON-NLS-2$
+				e.printStackTrace();
+			}
+			return false;
+		} finally {
+			monitor.exitWrite(); // free write lock
+		}
+		return true;
+	}
+	
+	public String toString() {
+		return "saving index for " + this.indexPath; //$NON-NLS-1$
+	}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/indexing/SourceIndexer.java

@@ -0,0 +1,75 @@
+/*******************************************************************************
+ * Copyright (c) 2003 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ ******************************************************************************/
+/*
+ * Created on Jun 1, 2003
+ */
+package org.eclipse.cdt.internal.core.search.indexing;
+
+/**
+ * @author bgheorgh
+*/
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.cdt.core.parser.IParser;
+import org.eclipse.cdt.core.parser.ParserFactory;
+import org.eclipse.cdt.core.parser.ParserMode;
+import org.eclipse.cdt.internal.core.index.IDocument;
+
+/**
+ * A SourceIndexer indexes source files using the parser. The following items are indexed:
+ * Declarations:
+ * - Classes
+ * - Structs
+ * - Unions
+ * References:
+ * - Classes
+ * - Structs
+ * - Unions
+ */
+public class SourceIndexer extends AbstractIndexer {
+	
+	//TODO: Indexer, add additional file types
+	public static final String[] FILE_TYPES= new String[] {"cpp","h"}; //$NON-NLS-1$
+	//protected DefaultProblemFactory problemFactory= new DefaultProblemFactory(Locale.getDefault());
+	IFile resourceFile;
+		
+	SourceIndexer(IFile resourceFile)	{
+		this.resourceFile = resourceFile;
+	}
+	/**
+	 * Returns the file types the <code>IIndexer</code> handles.
+	 */
+	public String[] getFileTypes(){
+		return FILE_TYPES;
+	}
+	
+	protected void indexFile(IDocument document) throws IOException {
+		// Add the name of the file to the index
+		output.addDocument(document);
+		// Create a new Parser
+		SourceIndexerRequestor requestor = new SourceIndexerRequestor(this, document);
+		IParser parser = ParserFactory.createParser( 
+							ParserFactory.createScanner( new StringReader( document.getStringContent() ), document.getName(), null, null, ParserMode.QUICK_PARSE ), 
+							requestor, ParserMode.QUICK_PARSE);
+		try{
+			parser.parse();
+		}
+		catch( Exception e ){
+			System.out.println( "Parse Exception in SourceIndexer" ); 
+			e.printStackTrace();
+		}
+	}
+	/**
+	 * Sets the document types the <code>IIndexer</code> handles.
+	 */
+	
+	public void setFileTypes(String[] fileTypes){}
+}

core/org.eclipse.cdt.core/index/org/eclipse/cdt/internal/core/search/messages.properties

@@ -0,0 +1,16 @@
+###############################################################################
+# Copyright (c) 2000, 2003 IBM Corporation and others.
+# All rights reserved. This program and the accompanying materials 
+# are made available under the terms of the Common Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/cpl-v10.html
+# 
+# Contributors:
+#     IBM Corporation - initial API and implementation
+###############################################################################
+### Eclipse CDT Core Search messages.
+
+engine.searching = Searching...
+exception.wrongFormat = Wrong format
+process.name = CDT Indexer
+manager.filesToIndex = {0} files to index

core/org.eclipse.cdt.core/model/org/eclipse/cdt/core/model/CoreModel.java

@@ -8,6 +8,7 @@ package org.eclipse.cdt.core.model;
 import org.eclipse.cdt.core.CCorePlugin;
 import org.eclipse.cdt.internal.core.model.BatchOperation;
 import org.eclipse.cdt.internal.core.model.CModelManager;
+import org.eclipse.cdt.internal.core.search.indexing.IndexManager;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
 import org.eclipse.core.resources.IProject;
@@ -215,4 +216,14 @@ public class CoreModel {
 			workspace.run(new BatchOperation(action), monitor);
 		}
 	}
+	
+	public void startIndexing()
+	{
+		manager.getIndexManager().reset();	
+	}
+	
+	public IndexManager getIndexManager(){
+		return manager.getIndexManager();
+	}
+	
 }

core/org.eclipse.cdt.core/model/org/eclipse/cdt/internal/core/model/CModelManager.java

@@ -41,6 +41,7 @@ import org.eclipse.core.resources.ResourcesPlugin;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.cdt.internal.core.search.indexing.IndexManager;
 
 public class CModelManager implements IResourceChangeListener {
 
@@ -755,4 +756,8 @@ public class CModelManager implements IResourceChangeListener {
 		}
 	}
 	
+	public IndexManager getIndexManager() {
+		return this.fDeltaProcessor.indexManager;
+	}
+	
 }

core/org.eclipse.cdt.core/model/org/eclipse/cdt/internal/core/model/DeltaProcessor.java

@@ -19,6 +19,8 @@ import org.eclipse.core.resources.IResource;
 import org.eclipse.core.resources.IResourceDelta;
 import org.eclipse.core.runtime.IPath;
 
+import org.eclipse.cdt.internal.core.search.indexing.IndexManager;
+
 /**
  * This class is used by <code>CModelManager</code> to convert
  * <code>IResourceDelta</code>s into <code>ICElementDelta</code>s.
@@ -32,6 +34,8 @@ public class DeltaProcessor {
 	 */
 	protected CElementDelta fCurrentDelta;
 	
+	protected IndexManager indexManager = new IndexManager();
+
 	/* The C element that was last created (see createElement(IResource). 
 	 * This is used as a stack of C elements (using getParent() to pop it, and 
 	 * using the various get*(...) to push it. */
@@ -333,6 +337,7 @@ public class DeltaProcessor {
 			// get the workspace delta, and start processing there.
 			IResourceDelta[] deltas = changes.getAffectedChildren();
 			ICElementDelta[] translatedDeltas = new CElementDelta[deltas.length];
+			System.out.println("delta.length: " + deltas.length);
 			for (int i = 0; i < deltas.length; i++) {
 				IResourceDelta delta = deltas[i];
 				fCurrentDelta = new CElementDelta(root);
@@ -446,9 +451,37 @@ public class DeltaProcessor {
 
 	protected void updateIndexAddResource(ICElement element, IResourceDelta delta) {
 		//CModelManager.getDefault().getIndexManager().addResource(delta.getResource());
+	
+		if (indexManager == null)
+			return;
+
+	    switch (element.getElementType()) {
+			case ICElement.C_PROJECT :
+					this.indexManager.indexAll(element.getCProject().getProject());
+					break;
+	    }
+		
 	}
 
 	protected void updateIndexRemoveResource(ICElement element, IResourceDelta delta) {
 		//CModelManager.getDefault().getIndexManager().removeResource(delta.getResource());
+	
+	/*
+		if (indexManager == null)
+						return;
+
+		switch (element.getElementType()) {
+			case ICElement.C_PROJECT :
+						this.indexManager.removeIndexFamily(element.getCProject().getProject().getFullPath());
+						// NB: Discarding index jobs belonging to this project was done during PRE_DELETE
+						break;
+						// NB: Update of index if project is opened, closed, or its java nature is added or removed
+						//     is done in updateCurrentDeltaAndIndex
+						
+		}
+		*/
 	}
-}
+	
+
+	}
+

core/org.eclipse.cdt.core/search/ChangeLog

@@ -0,0 +1,10 @@
+2003-06-25	Bogdan Gheorghe
+
+Modified:
+	
+	* search/org/eclipse/cdt/core/search/ICSearchConstants.java
+	* search/org/eclipse/cdt/internal/core/search/Utils.java
+	 - moved to index/org/eclipse/cdt/internal/core/search/Utils.java
+	* search/org/eclipse/cdt/internal/core/search/matching/CSearchPattern.java
+	* search/org/eclipse/cdt/internal/core/search/processing/IJob.java
+	* search/org/eclipse/cdt/internal/core/search/processing/JobManager.java

core/org.eclipse.cdt.core/search/org/eclipse/cdt/core/search/ICSearchConstants.java

@@ -9,15 +9,24 @@
  *     IBM Corp. - Rational Software - initial implementation
  ******************************************************************************/
 /*
- * Created on Jun 11, 2003
+ * Created on May 31, 2003
  */
 package org.eclipse.cdt.core.search;
 
 /**
- * @author aniefer
+ * @author bgheorgh
- *
+ */
- * To change the template for this generated type comment go to
+import org.eclipse.cdt.internal.core.search.processing.*;
- * Window>Preferences>Java>Code Generation>Code and Comments
+
+
+/**
+ * <p>
+ * This interface defines the constants used by the search engine.
+ * </p>
+ * <p>
+ * This interface declares constants only; it is not intended to be implemented.
+ * </p>
+ * @see org.eclipse.cdt.core.search.SearchEngine
  */
 public interface ICSearchConstants {
 	/**
@@ -34,36 +43,55 @@ public interface ICSearchConstants {
 	int TYPE= 0;
 
 	/**
-	 * The searched element is a method.
+	 * The searched element is a function.
 	 */
-	int METHOD= 1;
+	int FUNCTION= 1;
 
 	/**
-	 * The searched element is a package.
+	* The searched element is a namespace.
-	 */
+    */
-	//int PACKAGE= 2;
+	int NAMESPACE= 2;
 	
 	/**
 	 * The searched element is a constructor.
 	 */
-	int CONSTRUCTOR= 2;
+	int CONSTRUCTOR= 3;
 
 	/**
-	 * The searched element is a field.
+	 * The searched element is a member.
+     */
+	int MEMBER= 4;
+	
+	/**
+	 * The searched element is a variable.
+	 * More selective than using TYPE
 	 */
-	int FIELD= 3;
+	int VAR= 5;
 
 	/**
 	 * The searched element is a class. 
 	 * More selective than using TYPE
 	 */
-	int CLASS= 4;
+	int CLASS= 6;
 
 	/**
-	 * The searched element is an interface.
+	 * The searched element is a struct.
 	 * More selective than using TYPE
 	 */
-	int INTERFACE= 5;
+	int STRUCT= 7;
+
+	/**
+	 * The searched element is a enum.
+	 * More selective than using TYPE
+	 */
+	int ENUM= 8;
+	
+	/**
+	 * The searched element is a union.
+	 * More selective than using TYPE
+	 */
+	int UNION= 9;
+	
 	
 	/* Nature of match */
 	
@@ -81,7 +109,7 @@ public interface ICSearchConstants {
 	 * rather exclusively search for classes implementing an interface, or interfaces 
 	 * extending an interface.
 	 */
-	int IMPLEMENTORS= 1;
+	int DEFINITIONS= 1;
 
 	/**
 	 * The search result is a reference.
@@ -100,23 +128,6 @@ public interface ICSearchConstants {
 	 */
 	int ALL_OCCURRENCES= 3;
 
-	/**
-	 * When searching for field matches, it will exclusively find read accesses, as
-	 * opposed to write accesses. Note that some expressions are considered both
-	 * as field read/write accesses: for example, x++; x+= 1;
-	 * 
-	 * @since 2.0
-	 */
-	int READ_ACCESSES = 4;
-	
-	/**
-	 * When searching for field matches, it will exclusively find write accesses, as
-	 * opposed to read accesses. Note that some expressions are considered both
-	 * as field read/write accesses: for example,  x++; x+= 1;
-	 * 
-	 * @since 2.0
-	 */
-	int WRITE_ACCESSES = 5;
 	
 	/* Syntactic match modes */
 	
@@ -156,17 +167,17 @@ public interface ICSearchConstants {
 	 * has not finished indexing the workspace. Results will more likely
 	 * not contain all the matches.
 	 */
-	//int FORCE_IMMEDIATE_SEARCH = IJob.ForceImmediate;
+	int FORCE_IMMEDIATE_SEARCH = IJob.ForceImmediate;
 	/**
 	 * The search operation throws an <code>org.eclipse.core.runtime.OperationCanceledException</code>
 	 * if the underlying indexer has not finished indexing the workspace.
 	 */
-	//int CANCEL_IF_NOT_READY_TO_SEARCH = IJob.CancelIfNotReady;
+	int CANCEL_IF_NOT_READY_TO_SEARCH = IJob.CancelIfNotReady;
 	/**
 	 * The search operation waits for the underlying indexer to finish indexing 
 	 * the workspace before starting the search.
 	 */
-	//int WAIT_UNTIL_READY_TO_SEARCH = IJob.WaitUntilReady;
+	int WAIT_UNTIL_READY_TO_SEARCH = IJob.WaitUntilReady;
 	
 
 }

core/org.eclipse.cdt.core/search/org/eclipse/cdt/internal/core/search/Util.java

@@ -1,41 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2003 IBM Corporation and others.
- * All rights reserved. This program and the accompanying materials 
- * are made available under the terms of the Common Public License v0.5 
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/cpl-v05.html
- * 
- * Contributors:
- *     IBM Corp. - Rational Software - initial implementation
- ******************************************************************************/
-/*
- * Created on Jun 13, 2003
- */
-package org.eclipse.cdt.internal.core.search;
-
-/**
- * @author aniefer
- *
- * To change the template for this generated type comment go to
- * Window>Preferences>Java>Code Generation>Code and Comments
- */
-public class Util {
-
-	/**
-	 * 
-	 */
-	public Util() {
-		super();
-		// TODO Auto-generated constructor stub
-	}
-
-	/**
-	 * @param string
-	 * @return
-	 */
-	public static String bind(String string) {
-		// TODO Auto-generated method stub
-		return null;
-	}
-
-}

core/org.eclipse.cdt.core/search/org/eclipse/cdt/internal/core/search/matching/CSearchPattern.java

@@ -52,15 +52,15 @@ public abstract class CSearchPattern
 			case ICSearchConstants.TYPE:
 				pattern = createTypePattern( patternString, limitTo, matchMode, caseSensitive );
 				break;
-			case ICSearchConstants.METHOD:
+			//case ICSearchConstants.METHOD:
-				pattern = createMethodPattern( patternString, limitTo, matchMode, caseSensitive );
+			//	pattern = createMethodPattern( patternString, limitTo, matchMode, caseSensitive );
-				break;
+			//	break;
 			case ICSearchConstants.CONSTRUCTOR:
 				pattern = createConstructorPattern( patternString, limitTo, matchMode, caseSensitive );
 				break;
-			case ICSearchConstants.FIELD:
+			//case ICSearchConstants.FIELD:
-				pattern = createFieldPattern( patternString, limitTo, matchMode, caseSensitive );
+			//	pattern = createFieldPattern( patternString, limitTo, matchMode, caseSensitive );
-				break;
+			//	break;
 		}
 		
 		return pattern;

core/org.eclipse.cdt.core/search/org/eclipse/cdt/internal/core/search/processing/IJob.java

@@ -9,19 +9,17 @@
  *     IBM Corp. - Rational Software - initial implementation
  ******************************************************************************/
 /*
- * Created on Jun 13, 2003
+ * Created on May 30, 2003
  */
 package org.eclipse.cdt.internal.core.search.processing;
 
+/**
+ * @author bgheorgh
+ */
 import org.eclipse.core.runtime.IProgressMonitor;
 
-/**
- * @author aniefer
- *
- * To change the template for this generated type comment go to
- * Window>Preferences>Java>Code Generation>Code and Comments
- */
 public interface IJob {
+	
 	/* Waiting policies */
 	int ForceImmediate = 1;
 	int CancelIfNotReady = 2;
@@ -32,7 +30,7 @@ public interface IJob {
 	boolean COMPLETE = true;
 
 	/**
-	 * Answer true if the job belongs to a given family (tag)
+	 * True if job belongs to the passed in jobFamily
 	 */
 	public boolean belongsTo(String jobFamily);
 	/**
@@ -48,4 +46,5 @@ public interface IJob {
 	 * Answer whether the job is ready to run.
 	 */
 	public boolean isReadyToRun();
+	
 }

core/org.eclipse.cdt.core/search/org/eclipse/cdt/internal/core/search/processing/JobManager.java

@@ -9,32 +9,426 @@
  *     IBM Corp. - Rational Software - initial implementation
  ******************************************************************************/
 /*
- * Created on Jun 13, 2003
+ * Created on May 30, 2003
  */
 package org.eclipse.cdt.internal.core.search.processing;
 
-/**
+import org.eclipse.core.runtime.IProgressMonitor;
- * @author aniefer
+import org.eclipse.core.runtime.OperationCanceledException;
- *
+import org.eclipse.core.runtime.SubProgressMonitor;
- * To change the template for this generated type comment go to
+import org.eclipse.cdt.internal.core.search.Util;
- * Window>Preferences>Java>Code Generation>Code and Comments
- */
-public class JobManager implements Runnable {
 
-	/**
+public abstract class JobManager implements Runnable {
-	 * 
+
-	 */
+	/* queue of jobs to execute */
-	public JobManager() {
+	protected IJob[] awaitingJobs = new IJob[10];
-		super();
+	protected int jobStart = 0;
-		// TODO Auto-generated constructor stub
+	protected int jobEnd = -1;
+	protected boolean executing = false;
+
+	/* background processing */
+	protected Thread thread;
+
+	/* flag indicating whether job execution is enabled or not */
+	private boolean enabled = true;
+
+	public static boolean VERBOSE = false;
+	/* flag indicating that the activation has completed */
+	public boolean activated = false;
+	
+	private int awaitingClients = 0;
+
+	public static void verbose(String log) {
+		System.out.println("(" + Thread.currentThread() + ") " + log); //$NON-NLS-1$//$NON-NLS-2$
 	}
 
-	/* (non-Javadoc)
+	/**
-	 * @see java.lang.Runnable#run()
+	 * Invoked exactly once, in background, before starting processing any job
+	 */
+	public void activateProcessing() {
+		this.activated = true;
+	}
+	/**
+	 * Answer the amount of awaiting jobs.
+	 */
+	public synchronized int awaitingJobsCount() {
+
+		// pretend busy in case concurrent job attempts performing before activated
+		if (!activated)
+			return 1;
+
+		return jobEnd - jobStart + 1;
+
+	}
+	/**
+	 * Answers the first job in the queue, or null if there is no job available
+	 * Until the job has completed, the job manager will keep answering the same job.
+	 */
+	public synchronized IJob currentJob() {
+
+		if (!enabled)
+			return null;
+
+		if (jobStart <= jobEnd) {
+			return awaitingJobs[jobStart];
+		}
+		return null;
+	}
+	
+	public synchronized void disable() {
+		enabled = false;
+		if (VERBOSE)
+			JobManager.verbose("DISABLING background indexing"); //$NON-NLS-1$
+	}
+	/**
+	 * Remove the index from cache for a given project.
+	 * Passing null as a job family discards them all.
+	 */
+	public void discardJobs(String jobFamily) {
+
+		if (VERBOSE)
+			JobManager.verbose("DISCARD   background job family - " + jobFamily); //$NON-NLS-1$
+
+		boolean wasEnabled = isEnabled();
+		try {
+			IJob currentJob;
+			// cancel current job if it belongs to the given family
+			synchronized(this){
+				currentJob = this.currentJob();
+				disable();
+			}
+			if (currentJob != null 
+					&& (jobFamily == null || currentJob.belongsTo(jobFamily))) {
+	
+				currentJob.cancel();
+			
+				// wait until current active job has finished
+				while (thread != null && executing){
+					try {
+						if (VERBOSE)
+							JobManager.verbose("-> waiting end of current background job - " + currentJob); //$NON-NLS-1$ //$NON-NLS-2$
+						Thread.sleep(50);
+					} catch(InterruptedException e){
+					}
+				}
+			}
+	
+			// flush and compact awaiting jobs
+			int loc = -1;
+			synchronized(this) {
+				for (int i = jobStart; i <= jobEnd; i++) {
+					currentJob = awaitingJobs[i];
+					awaitingJobs[i] = null;
+					if (!(jobFamily == null
+						|| currentJob.belongsTo(jobFamily))) { // copy down, compacting
+						awaitingJobs[++loc] = currentJob;
+					} else {
+						if (VERBOSE)
+							JobManager.verbose("-> discarding background job  - " + currentJob); //$NON-NLS-1$
+						currentJob.cancel();
+					}
+				}
+				jobStart = 0;
+				jobEnd = loc;
+			}
+		} finally {
+			if (wasEnabled)
+				enable();
+		}
+		if (VERBOSE)
+			JobManager.verbose("DISCARD   DONE with background job family - " + jobFamily); //$NON-NLS-1$
+	}
+	
+	public synchronized void enable() {
+		enabled = true;
+		if (VERBOSE)
+			JobManager.verbose("ENABLING  background indexing"); //$NON-NLS-1$
+	}
+	
+	public synchronized boolean isEnabled() {
+		return enabled;
+	}
+	/**
+	 * Advance to the next available job, once the current one has been completed.
+	 * Note: clients awaiting until the job count is zero are still waiting at this point.
+	 */
+	protected synchronized void moveToNextJob() {
+
+		//if (!enabled) return;
+
+		if (jobStart <= jobEnd) {
+			awaitingJobs[jobStart++] = null;
+			if (jobStart > jobEnd) {
+				jobStart = 0;
+				jobEnd = -1;
+			}
+		}
+	}
+	/**
+	 * When idle, give chance to do something
+	 */
+	protected void notifyIdle(long idlingTime) {
+	}
+	/**
+	 * This API is allowing to run one job in concurrence with background processing.
+	 * Indeed since other jobs are performed in background, resource sharing might be 
+	 * an issue.Therefore, this functionality allows a given job to be run without
+	 * colliding with background ones.
+	 * Note: multiple thread might attempt to perform concurrent jobs at the same time,
+	 *            and should synchronize (it is deliberately left to clients to decide whether
+	 *            concurrent jobs might interfere or not. In general, multiple read jobs are ok).
+	 *
+	 * Waiting policy can be:
+	 * 		IJobConstants.ForceImmediateSearch
+	 * 		IJobConstants.CancelIfNotReadyToSearch
+	 * 		IJobConstants.WaitUntilReadyToSearch
+	 *
+	 */
+	public boolean performConcurrentJob(
+		IJob searchJob,
+		int waitingPolicy,
+		IProgressMonitor progress) {
+
+		if (VERBOSE)
+			JobManager.verbose("STARTING  concurrent job - " + searchJob); //$NON-NLS-1$
+		if (!searchJob.isReadyToRun()) {
+			if (VERBOSE)
+				JobManager.verbose("ABORTED   concurrent job - " + searchJob); //$NON-NLS-1$
+			return IJob.FAILED;
+		}
+
+		int concurrentJobWork = 100;
+		if (progress != null)
+			progress.beginTask("", concurrentJobWork); //$NON-NLS-1$
+		boolean status = IJob.FAILED;
+		if (awaitingJobsCount() > 0) {
+			switch (waitingPolicy) {
+
+				case IJob.ForceImmediate :
+					if (VERBOSE)
+						JobManager.verbose("-> NOT READY - forcing immediate - " + searchJob);//$NON-NLS-1$
+					boolean wasEnabled = isEnabled();
+					try {
+						disable(); // pause indexing
+						status = searchJob.execute(progress == null ? null : new SubProgressMonitor(progress, concurrentJobWork));
+					} finally {
+						if (wasEnabled)
+							enable();
+					}
+					if (VERBOSE)
+						JobManager.verbose("FINISHED  concurrent job - " + searchJob); //$NON-NLS-1$
+					return status;
+					
+				case IJob.CancelIfNotReady :
+					if (VERBOSE)
+						JobManager.verbose("-> NOT READY - cancelling - " + searchJob); //$NON-NLS-1$
+					if (progress != null) progress.setCanceled(true);
+					if (VERBOSE)
+						JobManager.verbose("CANCELED concurrent job - " + searchJob); //$NON-NLS-1$
+					throw new OperationCanceledException();
+
+				case IJob.WaitUntilReady :
+					int awaitingWork;
+					IJob previousJob = null;
+					IJob currentJob;
+					IProgressMonitor subProgress = null;
+					int totalWork = this.awaitingJobsCount();
+					if (progress != null && totalWork > 0) {
+						subProgress = new SubProgressMonitor(progress, concurrentJobWork / 2);
+						subProgress.beginTask("", totalWork); //$NON-NLS-1$
+						concurrentJobWork = concurrentJobWork / 2;
+					}
+					int originalPriority = this.thread.getPriority();
+					try {
+						synchronized(this) {
+							
+							// use local variable to avoid potential NPE (see Bug 20435 NPE when searching java method)
+							Thread t = this.thread;
+							if (t != null) {
+								t.setPriority(Thread.currentThread().getPriority());
+							}
+							this.awaitingClients++;
+						}
+						while ((awaitingWork = awaitingJobsCount()) > 0) {
+							if (subProgress != null && subProgress.isCanceled())
+								throw new OperationCanceledException();
+							currentJob = currentJob();
+							// currentJob can be null when jobs have been added to the queue but job manager is not enabled
+							if (currentJob != null && currentJob != previousJob) {
+								if (VERBOSE)
+									JobManager.verbose("-> NOT READY - waiting until ready - " + searchJob);//$NON-NLS-1$
+								if (subProgress != null) {
+									subProgress.subTask(
+										Util.bind("manager.filesToIndex", Integer.toString(awaitingWork))); //$NON-NLS-1$
+									subProgress.worked(1);
+								}
+								previousJob = currentJob;
+							}
+							try {
+								Thread.sleep(50);
+							} catch (InterruptedException e) {
+							}
+						}
+					} finally {
+						synchronized(this) {
+							this.awaitingClients--;
+							
+							// use local variable to avoid potential NPE (see Bug 20435 NPE when searching java method)
+							Thread t = this.thread;
+							if (t != null) {
+								t.setPriority(originalPriority);
+							}
+						}
+					}
+					if (subProgress != null) {
+						subProgress.done();
+					}
+			}
+		}
+		status = searchJob.execute(progress == null ? null : new SubProgressMonitor(progress, concurrentJobWork));
+		if (progress != null) {
+			progress.done();
+		}
+		if (VERBOSE)
+			JobManager.verbose("FINISHED  concurrent job - " + searchJob); //$NON-NLS-1$
+		return status;
+	}
+	
+	public abstract String processName();
+	
+	public synchronized void request(IJob job) {
+		if (!job.isReadyToRun()) {
+			if (VERBOSE)
+				JobManager.verbose("ABORTED request of background job - " + job); //$NON-NLS-1$
+			return;
+		}
+
+		// append the job to the list of ones to process later on
+		int size = awaitingJobs.length;
+		if (++jobEnd == size) { // when growing, relocate jobs starting at position 0
+			jobEnd -= jobStart;
+			System.arraycopy(
+				awaitingJobs,
+				jobStart,
+				(awaitingJobs = new IJob[size * 2]),
+				0,
+				jobEnd);
+			jobStart = 0;
+		}
+		awaitingJobs[jobEnd] = job;
+		if (VERBOSE)
+			JobManager.verbose("REQUEST   background job - " + job); //$NON-NLS-1$
+
+	}
+	/**
+	 * Flush current state
+	 */
+	public void reset() {
+		if (VERBOSE)
+			JobManager.verbose("Reset"); //$NON-NLS-1$
+
+		if (thread != null) {
+			discardJobs(null); // discard all jobs
+		} else {
+			/* initiate background processing */
+			thread = new Thread(this, this.processName());
+			thread.setDaemon(true);
+			// less prioritary by default, priority is raised if clients are actively waiting on it
+			thread.setPriority(Thread.NORM_PRIORITY-1); 
+			thread.start();
+		}
+	}
+	/**
+	 * Infinite loop performing resource indexing
 	 */
 	public void run() {
-		// TODO Auto-generated method stub
 
+		long idlingStart = -1;
+		activateProcessing();
+		try {
+			while (this.thread != null) {
+				try {
+					IJob job;
+					if ((job = currentJob()) == null) {
+						if (idlingStart < 0)
+							idlingStart = System.currentTimeMillis();
+						notifyIdle(System.currentTimeMillis() - idlingStart);
+						Thread.sleep(500);
+						continue;
+					} else {
+						idlingStart = -1;
+					}
+					if (VERBOSE) {
+						JobManager.verbose(awaitingJobsCount() + " awaiting jobs"); //$NON-NLS-1$
+						JobManager.verbose("STARTING background job - " + job); //$NON-NLS-1$
+					}
+					try {
+						executing = true;
+						/*boolean status = */job.execute(null);
+						//if (status == FAILED) request(job);
+					} finally {
+						executing = false;
+						if (VERBOSE) {
+							JobManager.verbose("FINISHED background job - " + job); //$NON-NLS-1$
+						}
+						moveToNextJob();
+						if (this.awaitingClients == 0) {
+							Thread.sleep(50);
+						}
+					}
+				} catch (InterruptedException e) { // background indexing was interrupted
+				}
+			}
+		} catch (RuntimeException e) {
+			if (this.thread != null) { // if not shutting down
+				// log exception
+				org.eclipse.cdt.internal.core.model.Util.log(e, "Background Indexer Crash Recovery"); //$NON-NLS-1$
+				
+				// keep job manager alive
+				this.discardJobs(null);
+				this.thread = null;
+				this.reset(); // this will fork a new thread with no waiting jobs, some indexes will be inconsistent
+			}
+			throw e;
+		} catch (Error e) {
+			if (this.thread != null && !(e instanceof ThreadDeath)) {
+				// log exception
+				org.eclipse.cdt.internal.core.model.Util.log(e, "Background Indexer Crash Recovery"); //$NON-NLS-1$
+				
+				// keep job manager alive
+				this.discardJobs(null);
+				this.thread = null;
+				this.reset(); // this will fork a new thread with no waiting jobs, some indexes will be inconsistent
+			}
+			throw e;
+		}
+	}
+	/**
+	 * Stop background processing, and wait until the current job is completed before returning
+	 */
+	public void shutdown() {
+
+		disable();
+		discardJobs(null); // will wait until current executing job has completed
+		Thread thread = this.thread;
+		this.thread = null; // mark the job manager as shutting down so that the thread will stop by itself
+		try {
+			if (thread != null) { // see http://bugs.eclipse.org/bugs/show_bug.cgi?id=31858
+				thread.join();
+			}
+		} catch (InterruptedException e) {
+		}
+	}
+	
+	public String toString() {
+		StringBuffer buffer = new StringBuffer(10);
+		buffer.append("Enabled:").append(this.enabled).append('\n'); //$NON-NLS-1$
+		int numJobs = jobEnd - jobStart + 1;
+		buffer.append("Jobs in queue:").append(numJobs).append('\n'); //$NON-NLS-1$
+		for (int i = 0; i < numJobs && i < 15; i++) {
+			buffer.append(i).append(" - job["+i+"]: ").append(awaitingJobs[jobStart+i]).append('\n'); //$NON-NLS-1$ //$NON-NLS-2$
+		}
+		return buffer.toString();
 	}	
 
 }

core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/CCorePlugin.java

@@ -140,6 +140,9 @@ public class CCorePlugin extends Plugin {
 		fIndexModel = IndexModel.getDefault();
 		fIndexModel.startup();
 		
+		//Fired up the new indexer
+		fCoreModel.startIndexing();
+		
 		fDescriptorManager = new CDescriptorManager();
 		fDescriptorManager.startup();
 		

core/org.eclipse.cdt.ui/ChangeLog

@@ -13,6 +13,14 @@
 	Began structuring expressions and declarators in Parser for ISourceElementRequestor.  
 	Updated other packages to use new interfaces.  
 
+2003-06-25 Bogdan Gheorghe
+	Added a new checkbox to the Indexer tab of the C/C++ Project Settings
+	to allow the new indexer to be turned on or off. 
+	* src/org/eclipse/cdt/ui/wizards/IndexerBlock.java
+	
+	Modified the CSearchPage to work with the new CSearchConstants
+	* src/org/eclipse/cdt/internal/ui/search/CSearchPage.java
+	
 2003-06-24 Thomas Fletcher
 
 	- Proposals will now include additional help information with them

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/search/CSearchPage.java

@@ -161,8 +161,8 @@ public class CSearchPage extends DialogPage implements ISearchPage, ICSearchCons
 		};
 
 		fSearchFor[   TYPE    ].addSelectionListener(cElementInitializer);
-		fSearchFor[  METHOD   ].addSelectionListener(cElementInitializer);
+		fSearchFor[  FUNCTION   ].addSelectionListener(cElementInitializer);
-		fSearchFor[   FIELD   ].addSelectionListener(cElementInitializer);
+		fSearchFor[   MEMBER   ].addSelectionListener(cElementInitializer);
 		fSearchFor[CONSTRUCTOR].addSelectionListener(cElementInitializer);
 		//fSearchFor[  PACKAGE  ].addSelectionListener(cElementInitializer);
 
@@ -283,34 +283,34 @@ public class CSearchPage extends DialogPage implements ISearchPage, ICSearchCons
 	}
 	private void setLimitTo(int searchFor) {
 		fLimitTo[ DECLARATIONS    ].setEnabled( true );
-		fLimitTo[ IMPLEMENTORS    ].setEnabled( false);
+		//fLimitTo[ IMPLEMENTORS    ].setEnabled( false);
 		fLimitTo[ REFERENCES      ].setEnabled( true );			
 		fLimitTo[ ALL_OCCURRENCES ].setEnabled( true );
-		fLimitTo[ READ_ACCESSES   ].setEnabled( false);
+		//fLimitTo[ READ_ACCESSES   ].setEnabled( false);
-		fLimitTo[ WRITE_ACCESSES  ].setEnabled( false);
+		//fLimitTo[ WRITE_ACCESSES  ].setEnabled( false);
 		
-		if (!(searchFor == TYPE || searchFor == INTERFACE) && fLimitTo[IMPLEMENTORS].getSelection()) {
+//		if (!(searchFor == TYPE || searchFor == INTERFACE) && fLimitTo[IMPLEMENTORS].getSelection()) {
-			fLimitTo[ IMPLEMENTORS ].setSelection(false);
+//			fLimitTo[ IMPLEMENTORS ].setSelection(false);
-			fLimitTo[ REFERENCES   ].setSelection(true);
+//			fLimitTo[ REFERENCES   ].setSelection(true);
-		}
+//		}
-
+//
-		if (!(searchFor == FIELD) && (getLimitTo() == READ_ACCESSES || getLimitTo() == WRITE_ACCESSES)) {
+//		if (!(searchFor == FIELD) && (getLimitTo() == READ_ACCESSES || getLimitTo() == WRITE_ACCESSES)) {
-			fLimitTo[ getLimitTo()].setSelection(false);
+//			fLimitTo[ getLimitTo()].setSelection(false);
-			fLimitTo[ REFERENCES  ].setSelection(true);
+//			fLimitTo[ REFERENCES  ].setSelection(true);
-		}
+//		}
-
+//
-		switch (searchFor) {
+//		switch (searchFor) {
-			case TYPE:
+//			case TYPE:
-			case INTERFACE:
+//			case INTERFACE:
-				fLimitTo[ IMPLEMENTORS ].setEnabled(true);
+//				fLimitTo[ IMPLEMENTORS ].setEnabled(true);
-				break;
+//				break;
-			case FIELD:
+//			case FIELD:
-				fLimitTo[ READ_ACCESSES  ].setEnabled(true);
+//				fLimitTo[ READ_ACCESSES  ].setEnabled(true);
-				fLimitTo[ WRITE_ACCESSES ].setEnabled(true);
+//				fLimitTo[ WRITE_ACCESSES ].setEnabled(true);
-				break;
+//				break;
-			default :
+//			default :
-				break;
+//				break;
-		}
+//		}
 	}
 	
 	private Control createSearchFor(Composite parent) {

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/ui/wizards/IndexerBlock.java

@@ -7,6 +7,7 @@ package org.eclipse.cdt.ui.wizards;
 
 import org.eclipse.cdt.core.CCorePlugin;
 import org.eclipse.cdt.core.index.IndexModel;
+import org.eclipse.cdt.internal.core.search.indexing.IndexManager;
 import org.eclipse.cdt.utils.ui.swt.IValidation;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.runtime.IProgressMonitor;
@@ -18,6 +19,7 @@ import org.eclipse.swt.widgets.Composite;
 
 public class IndexerBlock implements IWizardTab {
 	private Button indexerSwitch;
+	private Button indexerSwitch2;
 	IProject project;
 	IValidation page;
 
@@ -39,8 +41,13 @@ public class IndexerBlock implements IWizardTab {
 
 		indexerSwitch = new Button(composite, SWT.CHECK | SWT.RIGHT);
 		indexerSwitch.setAlignment(SWT.LEFT);
-		indexerSwitch.setText("Enable indexing service for this project");
+		indexerSwitch.setText("Enable CTAGS indexing service for this project");
 		indexerSwitch.setSelection(indexer.isEnabled(project));
+		
+		indexerSwitch2 = new Button(composite, SWT.CHECK | SWT.RIGHT);
+		indexerSwitch2.setAlignment(SWT.LEFT);
+		indexerSwitch2.setText("Enable NEW indexing service for this project");
+		indexerSwitch2.setSelection(false);
 		return composite;
 	}
 
@@ -50,6 +57,9 @@ public class IndexerBlock implements IWizardTab {
 	public void doRun(IProject project, IProgressMonitor monitor) {
 		IndexModel indexer = CCorePlugin.getDefault().getIndexModel();			
 		indexer.setEnabled(project, indexerSwitch.getSelection());
+		
+		IndexManager newIndexer = CCorePlugin.getDefault().getCoreModel().getIndexManager();
+		newIndexer.setEnabled(project, indexerSwitch2.getSelection());
 	}
 
 
@@ -79,12 +89,21 @@ public class IndexerBlock implements IWizardTab {
 	 */
 	public void setVisible(boolean visible) {
 		IndexModel indexer = CCorePlugin.getDefault().getIndexModel();			
+		IndexManager newIndexer = CCorePlugin.getDefault().getCoreModel().getIndexManager();
 		
 		if (indexerSwitch != null) {
 			//indexerSwitch.setAlignment(SWT.LEFT);
 			//indexerSwitch.setText("Enable indexing service for this project");
 			indexerSwitch.setSelection(indexer.isEnabled(project));
 		}
+		
+		if (indexerSwitch2 != null) {
+			//indexerSwitch.setAlignment(SWT.LEFT);
+			//indexerSwitch.setText("Enable indexing service for this project");
+			indexerSwitch2.setSelection(newIndexer.isEnabled(project));
+		}
+		
+		
 	}
 
 }