From 3abcf010f815b22f78133c35248a51561753b939 Mon Sep 17 00:00:00 2001
From: Vivian Kong <vivkong@ca.ibm.com>
Date: Fri, 27 Apr 2007 21:34:26 +0000
Subject: [PATCH] for bug 169847 - standalone indexer

---
 .../core/indexer/ILanguageMapper.java         |  38 ++
 .../core/indexer/StandaloneFastIndexer.java   |  73 +++
 .../indexer/StandaloneFastIndexerTask.java    | 209 ++++++++
 .../core/indexer/StandaloneFullIndexer.java   |  88 +++
 .../indexer/StandaloneFullIndexerTask.java    | 179 +++++++
 ...StandaloneIndexBasedCodeReaderFactory.java |  96 ++++
 .../core/indexer/StandaloneIndexer.java       | 428 +++++++++++++++
 .../core/indexer/StandaloneIndexerTask.java   | 499 ++++++++++++++++++
 8 files changed, 1610 insertions(+)
 create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/ILanguageMapper.java
 create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexer.java
 create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexerTask.java
 create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFullIndexer.java
 create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFullIndexerTask.java
 create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexBasedCodeReaderFactory.java
 create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexer.java
 create mode 100644 core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexerTask.java

diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/ILanguageMapper.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/ILanguageMapper.java
new file mode 100644
index 00000000000..f65abffa310
--- /dev/null
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/ILanguageMapper.java
@@ -0,0 +1,38 @@
+/*******************************************************************************
+ * Copyright (c) 2007 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * IBM - Initial API and implementation
+ *******************************************************************************/
+ 
+package org.eclipse.cdt.internal.core.indexer;
+
+import org.eclipse.cdt.core.model.ILanguage;
+
+/**
+ * This mapper can be used for determining the ILanguage for a particular file.
+ * 
+ * A mapper is needed for standalone indexing when the ILanguage for a file is unknown.
+ * 
+ * <p>
+ * <strong>EXPERIMENTAL</strong>. This class or interface has been added as
+ * part of a work in progress. There is no guarantee that this API will work or
+ * that it will remain the same. Please do not use this API without consulting
+ * with the CDT team.
+ * </p>
+ * 
+ * @since 4.0
+ */
+public interface ILanguageMapper {
+	
+	/**
+	 * Returns the language of a file.
+	 * @param file - path of the file
+	 * @return the ILanguage of the file
+	 */
+	ILanguage getLanguage(String file);
+}
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexer.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexer.java
new file mode 100644
index 00000000000..ea19e4c5b04
--- /dev/null
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexer.java
@@ -0,0 +1,73 @@
+/*******************************************************************************
+ * Copyright (c) 2006, 2007 QNX Software Systems and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * QNX - Initial API and implementation
+ * Markus Schorn (Wind River Systems)
+ * IBM Corporation
+ *******************************************************************************/
+
+package org.eclipse.cdt.internal.core.indexer;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+import org.eclipse.cdt.core.index.IIndexLocationConverter;
+import org.eclipse.cdt.core.parser.IParserLogService;
+import org.eclipse.cdt.core.parser.IScannerInfo;
+import org.eclipse.cdt.internal.core.index.IIndexFragment;
+import org.eclipse.cdt.internal.core.index.IWritableIndexFragment;
+import org.eclipse.cdt.internal.core.index.WritableCIndex;
+import org.eclipse.cdt.internal.core.pdom.WritablePDOM;
+import org.eclipse.core.runtime.CoreException;
+
+/**
+ * A standalone tool for populating an index.  This indexer optimizes for 
+ * speed at the expense of accuracy.
+ * 
+ * <p>
+ * <strong>EXPERIMENTAL</strong>. This class or interface has been added as
+ * part of a work in progress. There is no guarantee that this API will work or
+ * that it will remain the same. Please do not use this API without consulting
+ * with the CDT team.
+ * </p>
+ * 
+ * @since 4.0
+ */
+public class StandaloneFastIndexer extends StandaloneIndexer{	
+	
+	/**
+	 * Construct a fast standalone indexer.
+	 * @param writableIndexFile - the file where the PDOM index is stored
+	 * @param converter - a converter used to convert between String locations and IIndexLocations
+	 * @param linkageFactoryMappings - all of the available IPDOMLinkageFactories the index can use during indexing
+	 * @param scanner - provides include paths and defined symbols
+	 * @param mapper - a mapper used to determine ICLanguage for a particular file
+	 * @param log - logger
+	 * @throws CoreException
+	 */
+	public StandaloneFastIndexer(File writableIndexFile, IIndexLocationConverter converter, Map linkageFactoryMappings,
+			IScannerInfo scanner, ILanguageMapper mapper, IParserLogService log) throws CoreException {
+		WritablePDOM pdom = new WritablePDOM(writableIndexFile, converter, linkageFactoryMappings);
+		fIndex = new WritableCIndex(
+				new IWritableIndexFragment[] { pdom },
+				new IIndexFragment[0]);		
+		fIndexAllFiles = false;
+		fScanner = scanner;
+		fMapper = mapper;
+		fLog = log;	
+	}
+	
+	/**
+	 * Create a delegate standalone indexing task
+	 */
+	protected StandaloneIndexerTask createTask(List added, List changed, List removed) {
+		return new StandaloneFastIndexerTask(this, added, changed, removed);
+	}
+
+}
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexerTask.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexerTask.java
new file mode 100644
index 00000000000..248fe1047b1
--- /dev/null
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFastIndexerTask.java
@@ -0,0 +1,209 @@
+/*******************************************************************************
+ * Copyright (c) 2006, 2007 QNX Software Systems and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * QNX - Initial API and implementation
+ * Markus Schorn (Wind River Systems)
+ * IBM Corporation
+ *******************************************************************************/
+
+package org.eclipse.cdt.internal.core.indexer;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit;
+import org.eclipse.cdt.core.index.IIndexFile;
+import org.eclipse.cdt.core.index.IIndexFileLocation;
+import org.eclipse.cdt.core.model.AbstractLanguage;
+import org.eclipse.cdt.core.parser.CodeReader;
+import org.eclipse.cdt.core.parser.IScannerInfo;
+import org.eclipse.cdt.internal.core.index.IWritableIndex;
+import org.eclipse.cdt.internal.core.index.IndexFileLocation;
+import org.eclipse.cdt.internal.core.index.IndexBasedCodeReaderFactory.CallbackHandler;
+import org.eclipse.cdt.internal.core.index.IndexBasedCodeReaderFactory.IndexFileInfo;
+import org.eclipse.core.filesystem.URIUtil;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IProgressMonitor;
+
+/**
+ * A task for index updates.
+ * 
+ * <p>
+ * <strong>EXPERIMENTAL</strong>. This class or interface has been added as
+ * part of a work in progress. There is no guarantee that this API will work or
+ * that it will remain the same. Please do not use this API without consulting
+ * with the CDT team.
+ * </p>
+ * 
+ * @since 4.0
+ */
+public class StandaloneFastIndexerTask extends StandaloneIndexerTask implements CallbackHandler {
+	private List fChanged = new ArrayList();
+	private List fRemoved = new ArrayList();
+	private IWritableIndex fIndex;
+	private StandaloneIndexBasedCodeReaderFactory fCodeReaderFactory;
+	private Map fIflCache;
+	private int fCurrentConfigHash= 0;
+
+	public StandaloneFastIndexerTask(StandaloneFastIndexer indexer, List added,
+			List changed, List removed) {
+		super(indexer);
+		fChanged.addAll(added);
+		fChanged.addAll(changed);
+		fRemoved.addAll(removed);
+		updateInfo(0, 0, fChanged.size() + fRemoved.size());
+	}
+
+	public void run(IProgressMonitor monitor) throws IOException{
+		long start = System.currentTimeMillis();
+		try {
+			setupIndexAndReaderFactory();
+			fIndex.acquireReadLock();
+			try {
+				registerTUsInReaderFactory(fChanged);
+
+				Iterator i= fRemoved.iterator();
+				while (i.hasNext()) {
+					if (monitor.isCanceled())
+						return;
+					String tu = (String) i.next();
+					removeTU(fIndex, getIndexFileLocation(tu), 1);
+					if (isValidSourceUnitName(tu)) {
+						updateInfo(1, 0, 0);
+					}
+					else {
+						updateInfo(0, 1, -1);
+					}
+				}
+
+				// separate headers
+				List headers= new ArrayList();
+				List sources= fChanged;
+				for (Iterator iter = fChanged.iterator(); iter.hasNext();) {
+					String tu = (String) iter.next();
+					if (!isValidSourceUnitName(tu)) {
+						headers.add(tu);
+						iter.remove();
+					}
+				}
+
+				parseTUs(fIndex, 1, sources, headers, monitor);
+				if (monitor.isCanceled()) {
+					return;
+				}	
+			}
+			finally {
+				fIndex.releaseReadLock();
+			}
+		} catch (CoreException e) {
+			e.printStackTrace();
+		} catch (InterruptedException e) {
+		}
+		traceEnd(start);
+	}
+
+	private void setupIndexAndReaderFactory() throws CoreException {
+		fIndex= fIndexer.getIndex();
+		fIndex.resetCacheCounters();
+		fIflCache = new HashMap/*<String,IIndexFileLocation>*/();
+		fCodeReaderFactory = new StandaloneIndexBasedCodeReaderFactory(fIndex, fIflCache);
+		fCodeReaderFactory.setCallbackHandler(this);
+	}
+
+	private void registerTUsInReaderFactory(Collection files) throws IOException, CoreException {
+		int removed= 0;
+		for (Iterator iter = files.iterator(); iter.hasNext();) {
+			String sourcePath = (String) iter.next();
+			String path = new File(sourcePath).getCanonicalPath();
+			IIndexFileLocation location = getIndexFileLocation(path);
+			IndexFileInfo info= fCodeReaderFactory.createFileInfo(location);
+			if (updateAll()) {
+				info.fRequested= IndexFileInfo.REQUESTED;
+			}
+			else if (updateChangedTimestamps() && isOutdated(location, info.fFile)) {
+				info.fRequested= IndexFileInfo.REQUESTED;
+			}			
+			else {
+				iter.remove();
+				removed++;
+			}
+		}
+		updateInfo(0, 0, -removed);
+		
+	}
+
+	protected IIndexFileLocation findLocation(String absolutePath) {
+		IIndexFileLocation result = (IIndexFileLocation) fIflCache.get(absolutePath); 
+		if(result==null) {
+			//Standalone indexing stores the absolute paths of files being indexed
+			result = new IndexFileLocation(URIUtil.toURI(absolutePath),absolutePath); 
+			fIflCache.put(absolutePath, result);
+		}
+		return result;
+	}
+
+	protected IASTTranslationUnit createAST(AbstractLanguage lang, CodeReader codeReader, IScannerInfo scanInfo, int options, IProgressMonitor pm) throws CoreException {
+		
+		// get the AST in a "Fast" way
+		IASTTranslationUnit ast= lang.getASTTranslationUnit(codeReader, scanInfo, fCodeReaderFactory, fIndex, options, fIndexer.getParserLog());
+		if (pm.isCanceled()) {
+			return null;
+		}
+		// Clear the macros
+		fCodeReaderFactory.clearMacroAttachements();
+		return ast;
+	}
+
+	protected boolean needToUpdate(IIndexFileLocation location, int confighash) throws CoreException {
+		if (super.needToUpdate(location, confighash)) {
+			// file is requested or is not yet indexed.
+			IndexFileInfo info= fCodeReaderFactory.createFileInfo(location);
+			return needToUpdate(info, confighash);
+		}
+		return false;
+	}
+	
+	public boolean needToUpdate(IndexFileInfo info) throws CoreException {
+		return needToUpdate(info, fCurrentConfigHash);
+	}
+	
+	private boolean needToUpdate(IndexFileInfo info, int confighash) throws CoreException {
+		if (info.fFile == null) {
+			return true;
+		}
+		if (confighash != 0 && info.fRequested == IndexFileInfo.REQUESTED_IF_CONFIG_CHANGED) {
+			int oldhash= info.fFile.getScannerConfigurationHashcode();
+			if (oldhash == 0 || oldhash==confighash) {
+				info.fRequested= IndexFileInfo.NOT_REQUESTED;
+				updateInfo(0, 0, -1);
+			}
+			else {
+				info.fRequested= IndexFileInfo.REQUESTED;
+			}
+		}
+		return info.fRequested != IndexFileInfo.NOT_REQUESTED;
+	}
+
+
+	protected boolean postAddToIndex(IIndexFileLocation path, IIndexFile file)
+			throws CoreException {
+		IndexFileInfo info= fCodeReaderFactory.createFileInfo(path);
+		info.fFile= file;
+		if (info.fRequested != IndexFileInfo.NOT_REQUESTED) {
+			info.fRequested= IndexFileInfo.NOT_REQUESTED;
+			return true;
+		}
+		return false;
+	}
+}
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFullIndexer.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFullIndexer.java
new file mode 100644
index 00000000000..e00de8a06a1
--- /dev/null
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFullIndexer.java
@@ -0,0 +1,88 @@
+/*******************************************************************************
+ * Copyright (c) 2006, 2007 QNX Software Systems and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * QNX - Initial API and implementation
+ * Markus Schorn (Wind River Systems)
+ * IBM Corporation
+ *******************************************************************************/
+ 
+package org.eclipse.cdt.internal.core.indexer;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+import org.eclipse.cdt.core.dom.ICodeReaderFactory;
+import org.eclipse.cdt.core.index.IIndexLocationConverter;
+import org.eclipse.cdt.core.parser.IParserLogService;
+import org.eclipse.cdt.core.parser.IScannerInfo;
+import org.eclipse.cdt.internal.core.index.IIndexFragment;
+import org.eclipse.cdt.internal.core.index.IWritableIndexFragment;
+import org.eclipse.cdt.internal.core.index.WritableCIndex;
+import org.eclipse.cdt.internal.core.pdom.WritablePDOM;
+import org.eclipse.core.runtime.CoreException;
+
+/**
+ * A standalone tool for populating an index.  This indexer optimizes for 
+ * accuracy so it may be slower than the StandaloneFastIndexer.
+ * 
+ * <p>
+ * <strong>EXPERIMENTAL</strong>. This class or interface has been added as
+ * part of a work in progress. There is no guarantee that this API will work or
+ * that it will remain the same. Please do not use this API without consulting
+ * with the CDT team.
+ * </p>
+ * 
+ * @since 4.0
+ */
+public class StandaloneFullIndexer extends StandaloneIndexer{
+	
+	private ICodeReaderFactory fCodeReaderFactory;
+	
+	/**
+	 * Create a full indexer.
+	 * @param writableIndexFile - the file where the PDOM index is stored
+	 * @param converter - a converter used to convert between String locations and IIndexLocations
+	 * @param linkageFactoryMappings - all of the available IPDOMLinkageFactories the index can use during indexing
+	 * @param scanner - provides include paths and defined symbols
+	 * @param mapper - a mapper used to determine ICLanguage for a particular file
+	 * @param log - logger
+	 * @param codeReaderFactory - factory that provides CodeReaders for files included
+	 *                    		  by the source code being parsed.
+	 * @throws CoreException
+	 */
+	public StandaloneFullIndexer(File writableIndexFile, IIndexLocationConverter converter, Map linkageFactoryMappings,
+			IScannerInfo scanner, ILanguageMapper mapper, IParserLogService log, ICodeReaderFactory codeReaderFactory) throws CoreException {
+		WritablePDOM pdom = new WritablePDOM(writableIndexFile, converter, linkageFactoryMappings);
+		fIndex = new WritableCIndex(
+				new IWritableIndexFragment[] { pdom },
+				new IIndexFragment[0]);		
+		fIndexAllFiles = false;
+		fScanner = scanner;
+		fMapper = mapper;
+		fCodeReaderFactory = codeReaderFactory;
+		fLog = log;
+	}
+	
+	/**
+	 * Returns the factory that provides CodeReaders for files included
+	 * by the source code being parsed.
+	 * @return
+	 */
+	public ICodeReaderFactory getCodeReaderFactory() {
+		return fCodeReaderFactory;
+	}
+	
+	/**
+	 * Creates a delegate standalone indexing task
+	 */
+	protected StandaloneIndexerTask createTask(List added, List changed, List removed) {
+		return new StandaloneFullIndexerTask(this, added, changed, removed);
+	}
+
+}
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFullIndexerTask.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFullIndexerTask.java
new file mode 100644
index 00000000000..7f34773363b
--- /dev/null
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneFullIndexerTask.java
@@ -0,0 +1,179 @@
+/*******************************************************************************
+ * Copyright (c) 2006, 2007 QNX Software Systems and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * QNX - Initial API and implementation
+ * Markus Schorn (Wind River Systems)
+ * IBM Corporation
+ *******************************************************************************/
+ 
+package org.eclipse.cdt.internal.core.indexer;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit;
+import org.eclipse.cdt.core.index.IIndexFile;
+import org.eclipse.cdt.core.index.IIndexFileLocation;
+import org.eclipse.cdt.core.model.AbstractLanguage;
+import org.eclipse.cdt.core.parser.CodeReader;
+import org.eclipse.cdt.core.parser.IScannerInfo;
+import org.eclipse.cdt.internal.core.index.IWritableIndex;
+import org.eclipse.cdt.internal.core.index.IndexFileLocation;
+import org.eclipse.core.filesystem.URIUtil;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IProgressMonitor;
+
+/**
+ * A task for index updates.
+ * 
+ * <p>
+ * <strong>EXPERIMENTAL</strong>. This class or interface has been added as
+ * part of a work in progress. There is no guarantee that this API will work or
+ * that it will remain the same. Please do not use this API without consulting
+ * with the CDT team.
+ * </p>
+ * 
+ * @since 4.0
+ */
+public class StandaloneFullIndexerTask extends StandaloneIndexerTask {
+	private final static Object REQUIRED= new Object();
+	private final static Object MISSING = new Object();
+	private final static Object SKIP=     new Object();
+	
+	private List fChanged = new ArrayList();
+	private List fRemoved = new ArrayList();
+	private IWritableIndex fIndex = null;
+	private Map filePathsToParse = new HashMap/*<IIndexFileLocation, Object>*/();
+	private Map fIflCache = new HashMap/*<String, IIndexFileLocation>*/();
+
+	public StandaloneFullIndexerTask(StandaloneFullIndexer indexer, List added,
+			List changed, List removed) {
+		super(indexer);
+		fChanged.addAll(added);
+		fChanged.addAll(changed);
+		fRemoved.addAll(removed);
+		updateInfo(0, 0, fChanged.size() + fRemoved.size());
+	}
+
+	public void run(IProgressMonitor monitor) throws IOException {
+		long start = System.currentTimeMillis();
+		try {
+			setupIndex();
+			registerTUsInReaderFactory(fChanged);
+			
+			// separate headers
+			List headers= new ArrayList();
+			List sources= fChanged;
+			for (Iterator iter = fChanged.iterator(); iter.hasNext();) {
+				String tu = (String) iter.next();
+				if (!isValidSourceUnitName(tu)) {
+					headers.add(tu);
+					iter.remove();
+				}
+			}
+					
+			Iterator i= fRemoved.iterator();
+			while (i.hasNext()) {
+				if (monitor.isCanceled())
+					return;
+				String tu = (String) i.next();
+				removeTU(fIndex, getIndexFileLocation(tu), 0);
+				if (isValidSourceUnitName(tu)) {
+					updateInfo(1, 0, 0);
+				}
+				else {
+					updateInfo(0, 1, -1);
+				}
+			}
+
+			fIndex.acquireReadLock();
+			try {
+				parseTUs(fIndex, 1, sources, headers, monitor);
+			}
+			finally {
+				fIndex.releaseReadLock();
+			}
+		} catch (CoreException e) {
+			e.printStackTrace();
+		} catch (InterruptedException e) {
+		}
+		traceEnd(start);
+	}
+
+	private void setupIndex() throws CoreException {		
+		fIndex = fIndexer.getIndex();
+		fIndex.resetCacheCounters();
+	}
+
+	private void registerTUsInReaderFactory(Collection/*<ITranslationUnit>*/ sources)
+			throws IOException, CoreException {
+		int removed= 0;
+		filePathsToParse= new HashMap/*<IIndexFileLocation, Object>*/();
+		for (Iterator iter = sources.iterator(); iter.hasNext();) {
+			String sourcePath = (String) iter.next();
+			String path = new File(sourcePath).getCanonicalPath();
+			IIndexFileLocation location = getIndexFileLocation(path);
+			if (updateAll()) {
+				filePathsToParse.put(location, REQUIRED);
+			}
+			else if (updateChangedTimestamps() && isOutdated(location, fIndex.getFile(location))) {
+				filePathsToParse.put(location, REQUIRED);
+			}
+			else {
+				iter.remove();
+				removed++;
+				continue;
+			}
+			updateInfo(0, 0, -removed);
+		}
+	}
+
+	protected IIndexFileLocation findLocation(String absolutePath) {
+		IIndexFileLocation result = (IIndexFileLocation) fIflCache.get(absolutePath); 
+		if(result==null) {
+			//Standalone indexing stores the absolute paths of files being indexed
+			result = new IndexFileLocation(URIUtil.toURI(absolutePath),absolutePath); 
+			fIflCache.put(absolutePath, result);
+		}
+		return result;		
+	}
+
+	protected IASTTranslationUnit createAST(AbstractLanguage lang, CodeReader codeReader, IScannerInfo scanInfo, int options, IProgressMonitor pm) throws CoreException {		
+		// get the AST in a "Fast" way
+		IASTTranslationUnit ast= lang.getASTTranslationUnit(codeReader, scanInfo, ((StandaloneFullIndexer)fIndexer).getCodeReaderFactory(), null, options, fIndexer.getParserLog());
+		if (pm.isCanceled()) {
+			return null;
+		}		
+		return ast;
+	}
+
+	protected boolean needToUpdate(IIndexFileLocation location, int confighash) throws CoreException {
+		if (super.needToUpdate(location, confighash)) {
+			Object required= filePathsToParse.get(location);
+			if (required == null) {
+				required= MISSING;
+				filePathsToParse.put(location, required);
+			}			
+			return required != SKIP;
+		}
+		return false;
+	}
+
+	protected boolean postAddToIndex(IIndexFileLocation location, IIndexFile file)
+			throws CoreException {
+		Object required= filePathsToParse.get(location);
+		filePathsToParse.put(location, SKIP);
+		return required == REQUIRED;
+	}
+}
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexBasedCodeReaderFactory.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexBasedCodeReaderFactory.java
new file mode 100644
index 00000000000..b2a8a907b1c
--- /dev/null
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexBasedCodeReaderFactory.java
@@ -0,0 +1,96 @@
+/*******************************************************************************
+ * Copyright (c) 2007 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * IBM - Initial API and implementation
+ *******************************************************************************/
+ 
+package org.eclipse.cdt.internal.core.indexer;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.eclipse.cdt.core.dom.ICodeReaderFactory;
+import org.eclipse.cdt.core.dom.IMacroCollector;
+import org.eclipse.cdt.core.index.IIndex;
+import org.eclipse.cdt.core.index.IIndexFileLocation;
+import org.eclipse.cdt.core.parser.CodeReader;
+import org.eclipse.cdt.core.parser.ICodeReaderCache;
+import org.eclipse.cdt.internal.core.index.IndexBasedCodeReaderFactory;
+import org.eclipse.cdt.internal.core.index.IndexFileLocation;
+import org.eclipse.core.filesystem.URIUtil;
+
+/**
+ * A factory for CodeReaders construction.
+ * 
+ * <p>
+ * <strong>EXPERIMENTAL</strong>. This class or interface has been added as
+ * part of a work in progress. There is no guarantee that this API will work or
+ * that it will remain the same. Please do not use this API without consulting
+ * with the CDT team.
+ * </p>
+ * 
+ * @since 4.0
+ */
+public class StandaloneIndexBasedCodeReaderFactory extends IndexBasedCodeReaderFactory {
+
+	public static class DefaultFallBackFactory implements ICodeReaderFactory {
+
+		public CodeReader createCodeReaderForInclusion(IMacroCollector callback, String path) {
+			try {
+				if (!new File(path).exists())
+					return null;
+				return new CodeReader(path);
+			} catch (IOException e) {
+				return null;
+			}
+		}
+
+		public CodeReader createCodeReaderForTranslationUnit(String path) {
+			try {
+				if (!new File(path).exists())
+					return null;
+				return new CodeReader(path);
+			} catch (IOException e) {
+				return null;
+			}
+		}
+
+		public ICodeReaderCache getCodeReaderCache() {
+			return null;
+		}
+
+		public int getUniqueIdentifier() {
+			return 0;
+		}
+		
+	}
+	
+	public StandaloneIndexBasedCodeReaderFactory(IIndex index) {
+		super(index);
+	}
+	
+	public StandaloneIndexBasedCodeReaderFactory(IIndex index, ICodeReaderFactory fallbackFactory) {
+		super(index, new HashMap/*<String,IIndexFileLocation>*/(), fallbackFactory);
+	}
+	
+	public StandaloneIndexBasedCodeReaderFactory(IIndex index, Map iflCache) {
+		super(index, iflCache, new DefaultFallBackFactory());
+	}
+
+	public IIndexFileLocation findLocation(String absolutePath) {
+		IIndexFileLocation result = (IIndexFileLocation) getIFLCache().get(absolutePath); 
+		if(result==null) {
+			//Standalone indexing stores the absolute paths of files being indexed
+			result = new IndexFileLocation(URIUtil.toURI(absolutePath),absolutePath); 
+			getIFLCache().put(absolutePath, result);
+		}
+		return result;
+	}
+}
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexer.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexer.java
new file mode 100644
index 00000000000..2e12e2048e6
--- /dev/null
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexer.java
@@ -0,0 +1,428 @@
+/*******************************************************************************
+ * Copyright (c) 2007 Wind River Systems, Inc. and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *    Markus Schorn - initial API and implementation
+ * 	  IBM Corporation
+ *******************************************************************************/ 
+
+package org.eclipse.cdt.internal.core.indexer;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.eclipse.cdt.core.parser.IParserLogService;
+import org.eclipse.cdt.core.parser.IScannerInfo;
+import org.eclipse.cdt.internal.core.index.IWritableIndex;
+import org.eclipse.cdt.internal.core.pdom.IndexerProgress;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IProgressMonitor;
+
+/**
+ * The base class for standalone index population tools.
+ * 
+ * <p>
+ * <strong>EXPERIMENTAL</strong>. This class or interface has been added as
+ * part of a work in progress. There is no guarantee that this API will work or
+ * that it will remain the same. Please do not use this API without consulting
+ * with the CDT team.
+ * 
+ * This class is not thread safe.
+ * </p>
+ * 
+ * @since 4.0
+ */
+public abstract class StandaloneIndexer {
+	
+	/**
+	 * Parser should not skip any references.
+	 */
+	public static final int SKIP_NO_REFERENCES= 0;
+	
+	/**
+	 * Parser to skip all references.
+	 */
+	public static final int SKIP_ALL_REFERENCES= 1; 
+	
+	/**
+	 * Parser to skp type references.
+	 */
+	public static final int SKIP_TYPE_REFERENCES= 2;
+	
+	/**
+	 * Constant for indicating to update all translation units.
+	 */
+	public final static int UPDATE_ALL= 0x1;
+
+	/**
+	 * Constant for indicating to update translation units if their timestamp
+	 * has changed.
+	 */
+	public final static int UPDATE_CHECK_TIMESTAMPS= 0x2;
+		
+	/**
+	 * Empty list.
+	 */
+	protected static final List NO_TUS = new ArrayList();
+	/**
+	 * The IWritableIndex that stores all bindings and names.
+	 */
+	protected IWritableIndex fIndex;
+	
+	/**
+	 * A flag that indiciates if all files (sources without config, headers not included)
+	 * should be parsed.
+	 */
+	protected boolean fIndexAllFiles;
+	
+	/**
+	 * Collection of valid file extensions for C/C++ source.
+	 */
+	protected Set fValidSourceUnitNames;
+	
+	/**
+	 * The IScannerInfo that provides include paths and defined symbols.
+	 */
+	protected IScannerInfo fScanner;
+	
+	/**
+	 * The ILanguageMapper that determines the ILanguage for a file.
+	 */
+	protected ILanguageMapper fMapper;
+	
+	/**
+	 * The logger during parsing.
+	 */
+	protected IParserLogService fLog;
+	
+	/**
+	 * A flag that indicates if all activities during indexing should be shown.
+	 */
+	protected boolean fShowActivity;
+	
+	/**
+	 * A flag that indicates if any problems encountered during indexing.
+	 * should be shown.
+	 */
+	protected boolean fShowProblems;
+	
+	/**
+	 * A flag that indicates if statistics should be gathered during indexing.
+	 */
+	protected boolean fTraceStatistics;
+	
+	/**
+	 * The type of references the parser should skip.
+	 */
+	protected int fSkipReferences = SKIP_NO_REFERENCES;
+	
+	/**
+	 * The exclusion filter that skips over files that should not be indexed.
+	 */
+	protected FilenameFilter fExclusionFilter;
+	
+	/**
+	 * Files to parse up front.
+	 */
+	protected String[] fFilesToParseUpFront = new String[0];
+	
+	protected int fUpdateOptions = UPDATE_ALL;
+	
+	private IndexerProgress fProgress = null;
+	private volatile StandaloneIndexerTask fDelegate;
+	
+	private static FilenameFilter DEFAULT_FILTER = new FilenameFilter() {
+		public boolean accept(File dir, String name) {
+			return true;
+		}
+	};
+	
+	/**
+	 * Returns the index.
+	 * @return the IWritable index the indexer is writing to
+	 */
+	public IWritableIndex getIndex() {
+		return fIndex;
+	}
+	
+	/**
+	 * Returns true if all files (sources without config, headers not included)
+	 * should be parsed.  Otherwise, this method returns false.
+	 */
+	public boolean getIndexAllFiles() {
+		return fIndexAllFiles;
+	}
+	
+	/**
+	 * Returns the collection of valid file extensions for C/C++ source.
+	 * @return
+	 */
+	public Set getValidSourceUnitNames() {
+		return fValidSourceUnitNames;
+	}
+	
+	/**
+	 * Sets the collection of valid file extensions for C/C++ source.
+	 */
+	public void setValidSourceUnitNames(Set validSourceUnitNames) {
+		fValidSourceUnitNames = validSourceUnitNames;
+	}
+	
+	/**
+	 * Returns the IScannerInfo that provides include paths and defined symbols.
+	 * @return
+	 */
+	public IScannerInfo getScannerInfo() {
+		return fScanner;
+	}
+	
+	/**
+	 * Returns the ILanguageMapper that determines the ILanguage for a file.
+	 * @return
+	 */
+	public ILanguageMapper getLanguageMapper() {
+		return fMapper;
+	}
+	
+	/**
+	 * Returns the logger.
+	 * @return
+	 */
+	public IParserLogService getParserLog() {
+		return fLog;
+	}
+	
+	/**
+	 * Returns true if indexing activities should be shown.
+	 * Otherwise, this method returns false.
+	 * @return
+	 */
+	public boolean getShowActivity() {
+		return fShowActivity;
+	}
+	
+	/**
+	 * Tells indexer if indexing activities should be shown.
+	 */
+	public void setShowActivity(boolean showActivity) {
+		fShowActivity = showActivity;
+	}
+	
+	/**
+	 * Returns true if problems during indexing should be shown.
+	 * Otherwise, this method returns false.
+	 * @return
+	 */
+	public boolean getShowProblems() {
+		return fShowProblems;
+	}
+	
+	/**
+	 * Tells indexer if problems during indexing should be shown.
+	 */
+	public void setShowProblems(boolean showProblems) {
+		fShowProblems = showProblems;
+	}
+	
+	/**
+	 * Returns true if statistics should be gathered during indexing.
+	 * Otherwise, this method returns false..
+	 * @return
+	 */
+	public boolean getTraceStatistics() {
+		return fTraceStatistics;
+	}
+	
+	/**
+	 * Tells indexer if statistics should be gathered during indexing.
+	 */
+	public void setTraceStatistics(boolean traceStatistics) {
+		fTraceStatistics = traceStatistics;
+	}
+	
+	private IndexerProgress createProgress() {
+		IndexerProgress progress= new IndexerProgress();
+		progress.fTimeEstimate= 1000;
+		return progress;
+	}
+	
+	private void clearIndex() throws CoreException, InterruptedException {
+		IWritableIndex index= getIndex();
+		// First clear the pdom
+		index.acquireWriteLock(0);
+		try {
+			index.clear();
+		}
+		finally {
+			index.releaseWriteLock(0);
+		}
+	}
+	
+	/**
+	 * Returns the progress information.
+	 * @return
+	 */
+	public synchronized IndexerProgress getProgressInformation() {
+		return fDelegate != null ? fDelegate.getProgressInformation() : fProgress;
+	}
+	
+	/**
+	 * Returns the update options specified.
+	 * @return
+	 */
+	public int getUpdateOptions() {
+		return fUpdateOptions;
+	}
+	
+	/**
+	 * Specifies the update options, whether all translation units should be updated or only the ones
+	 * with timestamp changes.
+	 * @param options
+	 */
+	public void setUpdateOptions (int options) {
+		fUpdateOptions = options;
+	}
+		
+	/**
+	 * Clears the index and rebuild
+	 * @param tus - directories/files to be added to index
+	 * @param monitor
+	 * @throws IOException
+	 */
+	public void rebuild(List tus, IProgressMonitor monitor) throws IOException {
+		fProgress = createProgress();
+		
+		try {
+			clearIndex();
+			fDelegate= createTask(getFilesAdded(tus), NO_TUS, NO_TUS);
+			fDelegate.setUpdateFlags(fUpdateOptions);
+			
+		} catch (CoreException e) {
+			e.printStackTrace();
+		} catch (InterruptedException e) {
+		}
+		
+		if (fDelegate != null) {
+			fDelegate.run(monitor);
+		}
+		
+	}
+	
+	/**
+	 * Updates the index with changes.
+	 * @param added - directories/files to be added to the index
+	 * @param changed - files that have been changed
+	 * @param removed - files to be removed from the index
+	 * @param monitor
+	 * @throws IOException
+	 */
+	public void handleDelta(List added, List changed, List removed, IProgressMonitor monitor) throws IOException {
+		fProgress= new IndexerProgress();
+				
+		fDelegate= createTask(getFilesAdded(added), changed, removed);
+		if (fDelegate instanceof StandaloneIndexerTask) {
+			fDelegate.setUpdateFlags(fUpdateOptions);
+		}
+		
+		if (fDelegate != null) {
+			fDelegate.run(monitor);
+		}
+		
+	}
+	
+	/**
+	 * Returns files that are being added to the index, skipping over files that 
+	 * should not be excluded.
+	 * @param tus
+	 * @return
+	 */
+	private List getFilesAdded(List tus) {
+		List added = new ArrayList();
+		
+		FilenameFilter filter = getExclusionFilter();
+		if (filter == null) {
+			filter = DEFAULT_FILTER;
+		}
+		
+		Iterator iter = tus.iterator();
+		while (iter.hasNext()) {
+			String path = (String) iter.next();
+			File file = new File(path);
+			if (file.isDirectory()) {
+				String[] files = file.list(filter);
+				for (int i = 0; i < files.length; i++) {
+					added.add((String)files[i]);
+				}
+			}
+			else {				
+				if (filter.accept(file.getParentFile(), file.getName())) {
+					added.add(path);
+				}
+			}
+		}
+		return added;
+	}
+	
+	/**
+	 * Creates a delegate standalone indexing task
+	 */
+	protected abstract StandaloneIndexerTask createTask(List added, List changed, List removed);
+
+	/**
+	 * Return the type of references the parser should skip.
+	 * @return
+	 */
+	public int getSkipReferences() {
+		return fSkipReferences;
+	}
+	
+	/**
+	 * Sets the type of references the parser should skip.
+	 * @param skipReferences
+	 */
+	public void setSkipReferences(int skipReferences) {
+		fSkipReferences = skipReferences;
+	}
+
+	/**
+	 * Returns an array of files that should be parsed up front.
+	 * @return
+	 */
+	public String[] getFilesToParseUpFront() {
+		return fFilesToParseUpFront;
+	}
+	
+	/**
+	 * Sets an array of files that should be parsed up front.
+	 * @param filesToParseUpFront
+	 */
+	public void setFilesToParseUpFront(String[] filesToParseUpFront) {
+		fFilesToParseUpFront = filesToParseUpFront;
+	}
+	
+	/**
+	 * Returns the exclusion filter for this indexer.
+	 * @return
+	 */
+	public FilenameFilter getExclusionFilter() {
+		return fExclusionFilter;
+	}
+	
+	/**
+	 * Sets the exclusion filter that tells the indexer to skip over 
+	 * files that should not be indexed.
+	 * @param exclusionFilter
+	 */
+	public void setExclusionFilter(FilenameFilter exclusionFilter) {
+		fExclusionFilter = exclusionFilter;
+	}
+}
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexerTask.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexerTask.java
new file mode 100644
index 00000000000..d5c92ff089c
--- /dev/null
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/indexer/StandaloneIndexerTask.java
@@ -0,0 +1,499 @@
+/*******************************************************************************
+ * Copyright (c) 2006, 2007 Wind River Systems, Inc. and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *    Markus Schorn - initial API and implementation
+ *	  IBM Corporation
+ *******************************************************************************/ 
+
+package org.eclipse.cdt.internal.core.indexer;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.text.NumberFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.eclipse.cdt.core.CCorePlugin;
+import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit;
+import org.eclipse.cdt.core.index.IIndex;
+import org.eclipse.cdt.core.index.IIndexFile;
+import org.eclipse.cdt.core.index.IIndexFileLocation;
+import org.eclipse.cdt.core.index.IIndexInclude;
+import org.eclipse.cdt.core.model.AbstractLanguage;
+import org.eclipse.cdt.core.model.ILanguage;
+import org.eclipse.cdt.core.parser.CodeReader;
+import org.eclipse.cdt.core.parser.IExtendedScannerInfo;
+import org.eclipse.cdt.core.parser.IScannerInfo;
+import org.eclipse.cdt.internal.core.index.IIndexFragmentFile;
+import org.eclipse.cdt.internal.core.index.IWritableIndex;
+import org.eclipse.cdt.internal.core.index.IndexFileLocation;
+import org.eclipse.cdt.internal.core.pdom.IndexerProgress;
+import org.eclipse.cdt.internal.core.pdom.PDOMWriter;
+import org.eclipse.core.filesystem.URIUtil;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.Path;
+
+/**
+ * A task for index updates.
+ * 
+ * <p>
+ * <strong>EXPERIMENTAL</strong>. This class or interface has been added as
+ * part of a work in progress. There is no guarantee that this API will work or
+ * that it will remain the same. Please do not use this API without consulting
+ * with the CDT team.
+ * </p>
+ * 
+ * @since 4.0
+ */
+public abstract class StandaloneIndexerTask extends PDOMWriter{
+	private static final Object NO_CONTEXT = new Object();
+	
+	protected StandaloneIndexer fIndexer;
+	protected Map/*<IIndexFileLocation, Object>*/ fContextMap = new HashMap/*<IIndexFileLocation, Object>*/();
+	private List fFilesUpFront= new ArrayList();
+	private String fDummyFileName;
+	private URI fDummyFileURI;
+	private int fUpdateFlags= StandaloneIndexer.UPDATE_ALL;
+
+	protected StandaloneIndexerTask(StandaloneIndexer indexer) {
+		fIndexer= indexer;
+		setShowActivity(fIndexer.getShowActivity());
+		setShowProblems(fIndexer.getShowProblems());
+		setSkipReferences(fIndexer.getSkipReferences());
+	}
+	
+	/**
+	 * Return the indexer.
+	 * @return
+	 */
+	final public StandaloneIndexer getIndexer() {
+		return fIndexer;
+	}
+	
+	/**
+	 * Return indexer's progress information.
+	 */
+	final public IndexerProgress getProgressInformation() {
+		return super.getProgressInformation();
+	}
+	
+	public void setUpdateFlags(int flags) {
+		fUpdateFlags= flags;
+	}
+	
+	final public boolean updateAll() {
+		return (fUpdateFlags & StandaloneIndexer.UPDATE_ALL) != 0;
+	}
+	
+	final public boolean updateChangedTimestamps() {
+		return (fUpdateFlags & StandaloneIndexer.UPDATE_CHECK_TIMESTAMPS) != 0;
+	}
+	
+	/**
+	 * Tells the parser which files to parse first
+	 */
+	final public void setParseUpFront() {
+		String[] files = fIndexer.getFilesToParseUpFront();
+		for (int i = 0; i < files.length; i++) {
+			fFilesUpFront.add((String) files[i]);
+		}
+	}
+	
+	/**
+	 * Figurues out whether all files (sources without config, headers not included)
+	 * should be parsed.
+	 * @since 4.0
+	 */
+	final protected boolean getIndexAllFiles() {
+		return getIndexer().getIndexAllFiles();
+	}
+		
+	private IASTTranslationUnit createAST(IIndexFileLocation location, IScannerInfo scannerInfo, int options, IProgressMonitor pm) throws IOException, CoreException {
+		String path = location.getFullPath();
+		if (path == null) {
+			return null;
+		}
+		ILanguage language = fIndexer.getLanguageMapper().getLanguage(path);
+		if (language == null)
+			return null;
+
+		CodeReader codeReader = new CodeReader(path);	
+		if (codeReader == null) {
+			return null;
+		}
+
+		return createAST((AbstractLanguage)language, codeReader, scannerInfo, options, pm);
+	}
+
+	/**
+	 * Called to create the ast for a translation unit or a pre-parsed file. 
+	 * May return <code>null</code>.
+	 * @see #parseTUs(IWritableIndex, int, Collection, Collection, IProgressMonitor)
+	 * @since 4.0
+	 */
+	abstract protected IASTTranslationUnit createAST(AbstractLanguage lang, CodeReader codeReader, IScannerInfo scanInfo, int options, IProgressMonitor pm) throws CoreException;
+
+	/**
+	 * Convenience method for subclasses, parses the files calling out to the methods 
+	 * {@link #createAST(IIndexFileLocation, IProgressMonitor)}, 
+	 * {@link #needToUpdate(IIndexFileLocation)}, 
+	 * {@link #addSymbols(IASTTranslationUnit, IWritableIndex, int, IProgressMonitor)}
+	 * {@link #postAddToIndex(IIndexFileLocation, IIndexFile)} and
+	 * {@link #findLocation(String)}
+	 * @since 4.0
+	 */
+	protected void parseTUs(IWritableIndex index, int readlockCount, Collection sources, Collection headers, IProgressMonitor monitor) throws IOException, CoreException, InterruptedException {
+		int options= 0;
+		if (fIndexer.getSkipReferences() == StandaloneIndexer.SKIP_ALL_REFERENCES) {
+			options |= AbstractLanguage.OPTION_SKIP_FUNCTION_BODIES;
+		}
+		for (Iterator iter = fFilesUpFront.iterator(); iter.hasNext();) {
+			String upfront= (String) iter.next();
+			parseUpFront(upfront, options, index, readlockCount, monitor);
+		}
+		
+		// sources first
+		for (Iterator iter = sources.iterator(); iter.hasNext();) {
+			if (monitor.isCanceled()) 
+				return;
+			String sourcePath = (String) iter.next();
+			String path = new File(sourcePath).getCanonicalPath();	
+			final IIndexFileLocation ifl = getIndexFileLocation(path);
+
+			if (needToUpdate(ifl, 0)) {
+				parseTU(ifl, options, index, readlockCount, monitor);
+			}
+		}
+
+		// headers with context
+		for (Iterator iter = headers.iterator(); iter.hasNext();) {
+			if (monitor.isCanceled()) 
+				return;
+			String sourcePath = (String) iter.next();
+			String path = new File(sourcePath).getCanonicalPath();	
+			final IIndexFileLocation location = getIndexFileLocation(path);
+			
+			if (!needToUpdate(location, 0)) {
+				iter.remove();
+			} 
+			else {
+				IIndexFileLocation context= findContext(index, location);
+				if (context != null) {
+					parseTU(context, options, index, readlockCount, monitor);
+				}
+			}
+		}
+
+		// headers without context
+		if (getIndexAllFiles()) {
+			for (Iterator iter = headers.iterator(); iter.hasNext();) {
+				if (monitor.isCanceled()) 
+					return;
+
+				String sourcePath = (String) iter.next();
+				String path = new File(sourcePath).getCanonicalPath();	
+				final IIndexFileLocation ifl = getIndexFileLocation(path);
+
+				if (!needToUpdate(ifl, 0)) {
+					iter.remove();
+				}
+				else {
+					parseTU(ifl, options, index, readlockCount, monitor);
+				}
+			}
+		}
+	}
+	
+	final protected boolean isOutdated(IIndexFileLocation ifl, IIndexFile indexFile) throws CoreException {
+		if (indexFile == null) {
+			return true;
+		}
+		File res = new File(ifl.getFullPath());
+		if (res != null) {
+			if (indexFile != null) {
+				if (res.lastModified() == indexFile.getTimestamp()) {
+					return false;
+				}
+			}
+			return true;
+		}
+		return false;
+	}
+
+
+	private void parseTU(IIndexFileLocation location, int options, IWritableIndex index, int readlockCount, IProgressMonitor pm) throws IOException, CoreException, InterruptedException {
+		String path = location.getFullPath();
+				
+		try {
+			// skip if no scanner info
+			IScannerInfo scanner= fIndexer.getScannerInfo();
+			if (scanner == null) {
+				updateInfo(0, 0, -1);
+			}
+			else {
+				final int configHash = computeHashCode(scanner);
+				if (needToUpdate(location, configHash)) {
+					if (fShowActivity) {
+						System.out.println("Indexer: parsing " + path); //$NON-NLS-1$
+					}
+					long start= System.currentTimeMillis();
+					IASTTranslationUnit ast= createAST(location, scanner, options, pm);
+					fStatistics.fParsingTime += System.currentTimeMillis()-start;
+					if (ast != null) {
+						addSymbols(ast, index, readlockCount, configHash, pm);
+					}
+				}
+			}
+		}
+		catch (CoreException e) {
+			e.printStackTrace();
+		}
+		catch (RuntimeException e) {
+			e.printStackTrace();
+		}
+		catch (Error e) {
+			e.printStackTrace();
+		}
+	}
+	
+	private void parseUpFront(String file, int options, IWritableIndex index, int readlockCount, IProgressMonitor pm) throws CoreException, InterruptedException {
+		file= file.trim();
+		if (file.length() == 0) {
+			return;
+		}
+		try {
+			if (fShowActivity) {
+				System.out.println("Indexer: parsing " + file + " up front"); //$NON-NLS-1$ //$NON-NLS-2$
+			}
+			
+			long start= System.currentTimeMillis();
+
+			IASTTranslationUnit ast= null;
+			ILanguage l = fIndexer.getLanguageMapper().getLanguage(file);
+			if (l instanceof AbstractLanguage) {
+				AbstractLanguage lang= (AbstractLanguage) l;
+				IScannerInfo scanInfo = fIndexer.getScannerInfo();
+				String code= "#include \"" + file + "\"\n"; //$NON-NLS-1$ //$NON-NLS-2$
+				if (fDummyFileName == null) {
+					fDummyFileName= file + "___"; //$NON-NLS-1$
+					fDummyFileURI= findLocation(fDummyFileName).getURI();
+				}
+				CodeReader codeReader= new CodeReader(fDummyFileName, code.toCharArray());
+				ast= createAST(lang, codeReader, scanInfo, options, pm);
+			}
+			fStatistics.fParsingTime += System.currentTimeMillis()-start;
+			if (ast != null) {
+				addSymbols(ast, index, readlockCount, 0, pm);
+				updateInfo(-1, +1, 0);
+			}
+		}
+		catch (CoreException e) {
+			e.printStackTrace(); 
+		}
+		catch (RuntimeException e) {
+			e.printStackTrace();
+		}
+		catch (Error e) {
+			e.printStackTrace(); 
+		}
+	}
+
+	/**
+	 * Overriders must call super.needToUpdate(). If <code>false</code> is returned
+	 * this must be passed on to their caller:
+	 * <pre>
+	 *   if (super.needToUpdate()) {
+	 *      // your code
+	 *   }
+	 *   return false;
+	 */
+	protected boolean needToUpdate(IIndexFileLocation fileLoc, int configHash) throws CoreException {
+		return fDummyFileURI==null || !fDummyFileURI.equals(fileLoc.getURI());
+	}
+
+	private IIndexFileLocation findContext(IIndex index, IIndexFileLocation location) {
+		Object cachedContext= fContextMap.get(location);
+		if (cachedContext != null) {
+			return cachedContext == NO_CONTEXT ? null : (IIndexFileLocation) cachedContext;
+		}
+		IIndexFileLocation context= null;
+		fContextMap.put(location, NO_CONTEXT); // prevent recursion
+		IIndexFile pdomFile;
+		try {
+			pdomFile = index.getFile(location);
+			if (pdomFile != null) {
+				IIndexInclude[] includedBy = index.findIncludedBy(pdomFile, IIndex.DEPTH_ZERO);
+				ArrayList/*<IIndexFileLocation>*/ paths= new ArrayList/*<IIndexFileLocation>*/(includedBy.length);
+				for (int i = 0; i < includedBy.length; i++) {
+					IIndexInclude include = includedBy[i];
+					IIndexFileLocation incLocation = include.getIncludedByLocation();
+					if (isValidSourceUnitName(incLocation.getFullPath())) {
+						context = incLocation;
+						if (context != null) {
+							fContextMap.put(location, context);
+							return context;
+						}
+					}
+					paths.add(incLocation);
+				}
+				for (Iterator/*<IIndexFileLocation>*/ iter = paths.iterator(); iter.hasNext();) {
+					IIndexFileLocation nextLevel = (IIndexFileLocation) iter.next();
+					context = findContext(index, nextLevel);
+					if (context != null) {
+						fContextMap.put(location, context);
+						return context;
+					}
+				}
+			}
+		} catch (CoreException e) {
+			CCorePlugin.log(e);
+		}
+		return null;
+	}
+
+	/**
+	 * Conveninence method for subclasses, removes a translation unit from the index.
+	 * @since 4.0
+	 */
+	protected void removeTU(IWritableIndex index, IIndexFileLocation ifl, int readlocks) throws CoreException, InterruptedException {
+		index.acquireWriteLock(readlocks);
+		try {
+			IIndexFragmentFile file = (IIndexFragmentFile) index.getFile(ifl);
+			if (file != null)
+				index.clearFile(file, null);
+		} finally {
+			index.releaseWriteLock(readlocks);
+		}
+	}
+
+	protected void traceEnd(long start) {
+		if (fIndexer.getTraceStatistics()) {
+			IndexerProgress info= getProgressInformation();
+			String name= getClass().getName();
+			name= name.substring(name.lastIndexOf('.')+1);
+
+			System.out.println(name + " "  //$NON-NLS-1$
+					+ " (" + info.fCompletedSources + " sources, "  //$NON-NLS-1$ //$NON-NLS-2$
+					+ info.fCompletedHeaders + " headers)"); //$NON-NLS-1$
+			
+			boolean allFiles= getIndexAllFiles();
+			boolean skipRefs= fIndexer.getSkipReferences() == StandaloneIndexer.SKIP_ALL_REFERENCES;
+			boolean skipTypeRefs= skipRefs || fIndexer.getSkipReferences() == StandaloneIndexer.SKIP_TYPE_REFERENCES;
+			System.out.println(name + " Options: "  //$NON-NLS-1$
+					+ "parseAllFiles=" + allFiles //$NON-NLS-1$
+					+ ",skipReferences=" + skipRefs //$NON-NLS-1$
+					+ ", skipTypeReferences=" + skipTypeRefs //$NON-NLS-1$
+					+ "."); //$NON-NLS-1$
+			
+			System.out.println(name + " Timings: "  //$NON-NLS-1$
+					+ (System.currentTimeMillis() - start) + " total, " //$NON-NLS-1$
+					+ fStatistics.fParsingTime + " parser, " //$NON-NLS-1$
+					+ fStatistics.fResolutionTime + " resolution, " //$NON-NLS-1$
+					+ fStatistics.fAddToIndexTime + " index update."); //$NON-NLS-1$
+			int sum= fStatistics.fDeclarationCount+fStatistics.fReferenceCount+fStatistics.fProblemBindingCount;
+			double problemPct= sum==0 ? 0.0 : (double) fStatistics.fProblemBindingCount / (double) sum;
+			NumberFormat nf= NumberFormat.getPercentInstance();
+			nf.setMaximumFractionDigits(2);
+			nf.setMinimumFractionDigits(2);
+			System.out.println(name + " Result: " //$NON-NLS-1$
+					+ fStatistics.fDeclarationCount + " declarations, " //$NON-NLS-1$
+					+ fStatistics.fReferenceCount + " references, " //$NON-NLS-1$
+					+ fStatistics.fErrorCount + " errors, " //$NON-NLS-1$
+					+ fStatistics.fProblemBindingCount + "(" + nf.format(problemPct) + ") problems.");  //$NON-NLS-1$ //$NON-NLS-2$
+						
+			IWritableIndex index = fIndexer.getIndex();
+			if (index != null) {
+				long misses= index.getCacheMisses();
+				long hits= index.getCacheHits();
+				long tries= misses+hits;
+				double missPct= tries==0 ? 0.0 : (double) misses / (double) tries;
+				System.out.println(name + " Cache: " //$NON-NLS-1$
+					+ hits + " hits, "  //$NON-NLS-1$
+					+ misses + "(" + nf.format(missPct)+ ") misses."); //$NON-NLS-1$ //$NON-NLS-2$
+			}
+		}
+	}
+	
+	public abstract void run(IProgressMonitor monitor) throws IOException;
+	
+	protected IIndexFileLocation getIndexFileLocation(String path) {
+		String absolutePath = new File(path).getAbsolutePath();
+		//Standalone indexing stores the absolute paths of files being indexed
+		return new IndexFileLocation(URIUtil.toURI(absolutePath),absolutePath); 
+	}
+	
+	protected boolean isValidSourceUnitName(String filename) {
+		IPath path = new Path(filename);
+		if (fIndexer.getValidSourceUnitNames() == null || fIndexer.getValidSourceUnitNames().size() == 0)
+			return true;
+		return fIndexer.getValidSourceUnitNames().contains(path.getFileExtension());
+	}
+	
+	protected long getLastModified(IIndexFileLocation location) {
+		return new File(location.getFullPath()).lastModified();
+	}
+	
+	protected static int computeHashCode(IScannerInfo scannerInfo) {
+		int result= 0;
+		Map macros= scannerInfo.getDefinedSymbols();
+		if (macros != null) {
+			for (Iterator i = macros.entrySet().iterator(); i.hasNext();) {
+				Map.Entry entry = (Map.Entry) i.next();
+				String key = (String) entry.getKey();
+				String value = (String) entry.getValue();
+				result= addToHashcode(result, key);
+				if (value != null && value.length() > 0) {
+					result= addToHashcode(result, value);
+				}
+			}
+		}
+		String[] a= scannerInfo.getIncludePaths();
+		if (a != null) {
+			for (int i = 0; i < a.length; i++) {
+				result= addToHashcode(result, a[i]);
+
+			}
+		}
+		if (scannerInfo instanceof IExtendedScannerInfo) {
+			IExtendedScannerInfo esi= (IExtendedScannerInfo) scannerInfo;
+			a= esi.getIncludeFiles();
+			if (a != null) {
+				for (int i = 0; i < a.length; i++) {
+					result= addToHashcode(result, a[i]);
+
+				}
+			}			
+			a= esi.getLocalIncludePath();
+			if (a != null) {
+				for (int i = 0; i < a.length; i++) {
+					result= addToHashcode(result, a[i]);
+
+				}
+			}		
+			a= esi.getMacroFiles();
+			if (a != null) {
+				for (int i = 0; i < a.length; i++) {
+					result= addToHashcode(result, a[i]);
+
+				}
+			}		
+		}
+		return result;
+	}
+
+	private static int addToHashcode(int result, String key) {
+		return result*31 + key.hashCode();
+	}
+}