fixes 149571, timestamps for files in PDOM; fixes 161715, refined strategy for indexers;

disables indexer for most of the test cases

core/org.eclipse.cdt.core/model/org/eclipse/cdt/core/model/CoreModelUtil.java

@@ -11,7 +11,10 @@
 
 package org.eclipse.cdt.core.model;
 
+import org.eclipse.cdt.core.CCorePlugin;
+import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IResource;
+import org.eclipse.core.resources.ResourcesPlugin;
 import org.eclipse.core.runtime.IPath;
 
 public class CoreModelUtil {
@@ -521,4 +524,61 @@ public class CoreModelUtil {
 		return -1;
 	}
 
+	/**
+	 * Searches for a translation unit within the cprojects. For external files the ones
+	 * from the given project are preferred.
+	 * @since 4.0
+	 */
+	public static ITranslationUnit findTranslationUnitForLocation(IPath location, ICProject preferredProject) throws CModelException {
+		IFile[] files= ResourcesPlugin.getWorkspace().getRoot().findFilesForLocation(location);
+		if (files.length > 0) {
+			for (int i = 0; i < files.length; i++) {
+				IFile file = files[i];
+				ITranslationUnit tu= findTranslationUnit(file);
+				if (tu != null) {
+					return tu;
+				}
+			}
+		}
+		else {
+			CoreModel coreModel = CoreModel.getDefault();
+			ITranslationUnit tu= null;
+			if (preferredProject != null) {
+				tu= coreModel.createTranslationUnitFrom(preferredProject, location);
+			}
+			if (tu == null) {
+				ICProject[] projects= coreModel.getCModel().getCProjects();
+				for (int i = 0; i < projects.length && tu == null; i++) {
+					ICProject project = projects[i];
+					if (!project.equals(preferredProject)) {
+						tu= coreModel.createTranslationUnitFrom(project, location);
+					}
+				}
+			}
+			return tu;
+		}
+		return null;
+	}	
+	
+	/**
+	 * Returns the translation unit for the file given or <code>null</code>.
+	 */
+    public static ITranslationUnit findTranslationUnit(IFile file) {
+		if (CoreModel.isTranslationUnit(file)) {
+			ICProject cp= CoreModel.getDefault().getCModel().getCProject(file.getProject().getName());
+			if (cp != null) {
+				ICElement tu;
+				try {
+					tu = cp.findElement(file.getProjectRelativePath());
+					if (tu instanceof ITranslationUnit) {
+						return (ITranslationUnit) tu;
+					}
+				} catch (CModelException e) {
+					CCorePlugin.log(e);
+				}
+			}
+		}
+        return null;
+    }
+
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/dom/IPDOMIndexer.java

@@ -35,4 +35,16 @@ public interface IPDOMIndexer {
 	 * @return the unique ID of type of this indexer
 	 */
 	public String getID();
+	
+	/**
+	 * Returns whether to index headers that are not actually included by any source.
+	 * @since 4.0
+	 */
+	public boolean getIndexAllHeaders();
+	
+	/**
+	 * Clients are not allowed to call this method, it is called by the framework. 
+	 * @since 4.0
+	 */
+	public void setIndexAllHeaders(boolean value);
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/index/IIndexFile.java

@@ -50,4 +50,11 @@ public interface IIndexFile {
 	 * @throws CoreException
 	 */
 	IIndexMacro[] getMacros() throws CoreException;
+	
+	/**
+	 * Last modification of file before it was indexed.
+	 * @return the last modification date of the file at the time it was parsed.
+	 * @throws CoreException 
+	 */
+	long getTimestamp() throws CoreException;
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/index/IIndexName.java

@@ -12,6 +12,7 @@
 package org.eclipse.cdt.core.index;
 
 import org.eclipse.cdt.core.dom.IName;
+import org.eclipse.core.runtime.CoreException;
 
 
 /**
@@ -35,6 +36,12 @@ public interface IIndexName extends IName {
 	 */
 	public String getFileName();
 
+	/**
+	 * Returns the file the name belongs to.
+	 * @throws CoreException 
+	 */
+	public IIndexFile getFile() throws CoreException;
+	
 	/**
 	 * Returns the character offset of the location of the name.
 	 */

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/index/IndexFilter.java

@@ -26,6 +26,8 @@ import org.eclipse.cdt.core.dom.ILinkage;
  */
 
 public class IndexFilter {
+	public static final IndexFilter ALL = new IndexFilter();
+
 	/**
 	 * Returns whether or not to include objects of the given linkage in the query.
 	 * @see IIndex#findBindings(java.util.regex.Pattern, boolean, IndexFilter, org.eclipse.core.runtime.IProgressMonitor)

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/CodeReader.java

@@ -27,7 +27,7 @@ import org.eclipse.cdt.core.parser.util.CharArrayUtils;
 public class CodeReader {
     public static final String SYSTEM_DEFAULT_ENCODING = System.getProperty( "file.encoding" ); //$NON-NLS-1$
 	private static final String NF = "<text>"; //$NON-NLS-1$
-	private static final char [] NOFILE = NF.toCharArray(); //$NON-NLS-1$
+	private static final char [] NOFILE = NF.toCharArray(); 
 	
 	public final char[] buffer;
 	public final char[] filename;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTName.java

@@ -10,13 +10,11 @@
  *******************************************************************************/
 package org.eclipse.cdt.internal.core.dom.parser.cpp;
 
-import org.eclipse.cdt.core.dom.ILinkage;
 import org.eclipse.cdt.core.dom.ast.ASTVisitor;
 import org.eclipse.cdt.core.dom.ast.IASTName;
 import org.eclipse.cdt.core.dom.ast.IASTNameOwner;
 import org.eclipse.cdt.core.dom.ast.IASTNode;
 import org.eclipse.cdt.core.dom.ast.IBinding;
-import org.eclipse.cdt.internal.core.dom.Linkage;
 
 /**
  * @author jcamelon

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPClassInstance.java

@@ -13,7 +13,6 @@
  */
 package org.eclipse.cdt.internal.core.dom.parser.cpp;
 
-import org.eclipse.cdt.core.dom.ILinkage;
 import org.eclipse.cdt.core.dom.ast.DOMException;
 import org.eclipse.cdt.core.dom.ast.IASTName;
 import org.eclipse.cdt.core.dom.ast.IBinding;
@@ -31,8 +30,6 @@ import org.eclipse.cdt.core.dom.ast.cpp.ICPPScope;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPTemplateInstance;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPTemplateTypeParameter;
 import org.eclipse.cdt.core.parser.util.ObjectMap;
-import org.eclipse.cdt.internal.core.dom.Linkage;
-import org.eclipse.core.runtime.CoreException;
 
 /**
  * @author aniefer

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/index/CIndex.java

@@ -33,7 +33,6 @@ import org.eclipse.cdt.core.model.ICElement;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
-import org.eclipse.core.runtime.Path;
 import org.eclipse.core.runtime.SubProgressMonitor;
 
 public class CIndex implements IIndex {
@@ -169,8 +168,9 @@ public class CIndex implements IIndex {
 
 	public IIndexFile getFile(IPath location) throws CoreException {
 		IIndexFile result= null;
+		String path= location.toOSString();
 		for (int i = 0; result==null && i < fPrimaryFragmentCount; i++) {
-			result= fFragments[i].getFile(location);
+			result= fFragments[i].getFile(path);
 		}
 		return result;
 	}
@@ -184,7 +184,7 @@ public class CIndex implements IIndex {
 				return result;
 			}
 		}
-		Path location= new Path(include.getIncludesLocation());
+		String location= include.getIncludesLocation();
 		for (int i = 0; i < fPrimaryFragmentCount; i++) {
 			IIndexFragment otherFrag = fFragments[i];
 			if (otherFrag != frag) {

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/index/IIndexFragment.java

@@ -18,7 +18,6 @@ import org.eclipse.cdt.core.dom.ast.IBinding;
 import org.eclipse.cdt.core.index.IIndex;
 import org.eclipse.cdt.core.index.IndexFilter;
 import org.eclipse.core.runtime.CoreException;
-import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
 
 /**
@@ -55,7 +54,7 @@ public interface IIndexFragment {
 	 * @return the file for the location
 	 * @throws CoreException
 	 */
-	IIndexFragmentFile getFile(IPath location) throws CoreException;
+	IIndexFragmentFile getFile(String location) throws CoreException;
 
 	/**
 	 * Returns all include directives that point to the given file. The input file may belong to 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/index/IIndexFragmentFile.java

@@ -12,6 +12,7 @@
 package org.eclipse.cdt.internal.core.index;
 
 import org.eclipse.cdt.core.index.IIndexFile;
+import org.eclipse.core.runtime.CoreException;
 
 public interface IIndexFragmentFile extends IIndexFile {
 
@@ -20,4 +21,10 @@ public interface IIndexFragmentFile extends IIndexFile {
 	 */
 	IIndexFragment getIndexFragment();
 
+	/**
+	 * Sets the timestamp of the file
+	 * @throws CoreException 
+	 */
+	void setTimestamp(long timestamp) throws CoreException;
+
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/index/IWritableIndex.java

@@ -15,6 +15,7 @@ import org.eclipse.cdt.core.dom.ast.IASTName;
 import org.eclipse.cdt.core.dom.ast.IASTPreprocessorMacroDefinition;
 import org.eclipse.cdt.core.index.IIndex;
 import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IPath;
 
 /**
  * Interface used by the indexer to write to the index. 
@@ -26,7 +27,7 @@ public interface IWritableIndex extends IIndex {
 	/**
 	 * Creates a file object for the given location or returns an existing one.
 	 */
-	IIndexFragmentFile addFile(String fileLocation) throws CoreException;
+	IIndexFragmentFile addFile(IPath fileLocation) throws CoreException;
 
 	/**
 	 * Adds an AST name to the given file.

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/index/IndexBasedCodeReaderFactory.java

@@ -15,17 +15,17 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import org.eclipse.cdt.core.CCorePlugin;
 import org.eclipse.cdt.core.dom.ICodeReaderFactory;
 import org.eclipse.cdt.core.index.IIndex;
 import org.eclipse.cdt.core.index.IIndexFile;
 import org.eclipse.cdt.core.index.IIndexInclude;
+import org.eclipse.cdt.core.model.ITranslationUnit;
 import org.eclipse.cdt.core.parser.CodeReader;
 import org.eclipse.cdt.core.parser.ICodeReaderCache;
 import org.eclipse.cdt.core.parser.IMacro;
@@ -41,14 +41,21 @@ import org.eclipse.core.runtime.Path;
  */
 public class IndexBasedCodeReaderFactory implements ICodeReaderFactory {
 
+	private final static boolean CASE_SENSITIVE_FILES= !new File("a").equals(new File("A"));  //$NON-NLS-1$//$NON-NLS-2$
 	private final IIndex index;
-	
+	private Map fileInfoCache = new HashMap(); // filename, fileInfo
-	private Map fileCache = new HashMap(); // filename, pdomFile
-	private Map macroCache = new HashMap();// record, list of IMacros
 	private List usedMacros = new ArrayList();
 	
 	private static final char[] EMPTY_CHARS = new char[0];
 	
+	public static class FileInfo {
+		private FileInfo() {}
+		public IIndexFile fFile= null;
+		public IMacro[] fMacros= null;
+		public boolean fNeedToIndex= false;
+	}
+	private static class NeedToParseException extends Exception {}
+	
 	public IndexBasedCodeReaderFactory(IIndex index) {
 		this.index = index;
 	}
@@ -61,68 +68,85 @@ public class IndexBasedCodeReaderFactory implements ICodeReaderFactory {
 		return ParserUtil.createReader(path, null);
 	}
 	
-	private void fillMacros(IIndexFragmentFile file, IScanner scanner, Set visited) throws CoreException {
-		Object key= file.getLocation();	// mstodo revisit, the pdom-id was faster but cannot be used in indexes.
-		if (!visited.add(key)) {
-			return;
-		}
-
-		// Follow the includes
-		IIndexInclude[] includeDirectives= file.getIncludes();
-		for (int i = 0; i < includeDirectives.length; i++) {
-			IIndexFragmentFile includedFile= (IIndexFragmentFile) index.resolveInclude(includeDirectives[i]);
-			if (includedFile != null) {
-				fillMacros(includedFile, scanner, visited);
-			}
-			// mstodo revisit, what if includedFile == null (problem with index??)
-		}
-		
-		// Add in my macros now
-		IMacro[] macros = (IMacro[]) macroCache.get(key);
-		if (macros == null) {
-			macros= file.getMacros();
-			macroCache.put(key, macros);
-		}
-
-		for (int i = 0; i < macros.length; ++i)
-			scanner.addDefinition(macros[i]);
-
-		// record the macros we used.
-		usedMacros.add(macros);
-	}
-	
 	public CodeReader createCodeReaderForInclusion(IScanner scanner, String path) {
-		// Don't parse inclusion if it is already captured
+		// if the file is in the index, we skip it
-		try {
+		File location= new File(path);
+		String canonicalPath= path;
+		if (!location.exists()) {
+			return null;
+		}
+		if (!CASE_SENSITIVE_FILES) {
 			try {
-				File file = new File(path);
+				canonicalPath= location.getCanonicalPath();
-				if (!file.exists())
-					return null;
-				path = file.getCanonicalPath();
-			} catch (IOException e) {
-				// ignore and use the path we were passed in
 			}
-			IIndexFragmentFile file = getCachedFile(path);
+			catch (IOException e) {
-			if (file == null) {
+				// just use the original
-				file = (IIndexFragmentFile) index.getFile(new Path(path));
+			}
-				if (file != null) {
+		}
-					addFileToCache(path, file);
+		try {
+			FileInfo info= createInfo(canonicalPath, null);
+
+			// try to build macro dictionary off index
+			if (info.fFile != null) {
+				try {
+					LinkedHashSet infos= new LinkedHashSet();
+					getInfosForMacroDictionary(info, infos);
+					for (Iterator iter = infos.iterator(); iter.hasNext();) {
+						FileInfo fi = (FileInfo) iter.next();
+						if (fi.fMacros == null) {
+							assert fi.fFile != null;
+							fi.fMacros= fi.fFile.getMacros();
+						}
+						for (int i = 0; i < fi.fMacros.length; ++i) {
+							scanner.addDefinition(fi.fMacros[i]);
+						}
+						// record the macros we used.
+						usedMacros.add(fi.fMacros);
+					}
+					return new CodeReader(canonicalPath, EMPTY_CHARS);
+				} catch (NeedToParseException e) {
 				}
 			}
-			if (file != null) {
-				// Already got things from here,
-				// add the macros to the scanner
-				fillMacros(file, scanner, new HashSet());
-				return new CodeReader(path, EMPTY_CHARS);
-			}
-		} catch (CoreException e) {
-			CCorePlugin.log(e); 
 		}
-		
+		catch (CoreException e) {
-		return ParserUtil.createReader(path, null);
+			CCorePlugin.log(e);
+			// still try to parse the file.
+		}
+
+		return ParserUtil.createReader(canonicalPath, null);
+	}
+
+	private FileInfo createInfo(String location, IIndexFile file) throws CoreException {
+		FileInfo info= (FileInfo) fileInfoCache.get(location);
+		if (info == null) {
+			info= new FileInfo();			
+			info.fFile= file == null ? index.getFile(new Path(location)) : file;
+			fileInfoCache.put(location, info);
+		}
+		return info;
 	}
 	
-	public void clearMacros() {
+	private void getInfosForMacroDictionary(FileInfo fileInfo, LinkedHashSet target) throws CoreException, NeedToParseException {
+		if (!target.add(fileInfo)) {
+			return;
+		}
+		if (fileInfo.fFile == null || fileInfo.fNeedToIndex) {
+			throw new NeedToParseException();
+		}
+		
+		// Follow the includes
+		IIndexFile file= fileInfo.fFile;
+		IIndexInclude[] includeDirectives= file.getIncludes();
+		for (int i = 0; i < includeDirectives.length; i++) {
+			IIndexFile includedFile= index.resolveInclude(includeDirectives[i]);
+			if (includedFile != null) {
+				FileInfo nextInfo= createInfo(includedFile.getLocation(), includedFile);
+				getInfosForMacroDictionary(nextInfo, target);
+			}
+		}
+	}
+	
+	public void clearMacroAttachements() {
 		Iterator i = usedMacros.iterator();
 		while (i.hasNext()) {
 			IMacro[] macros = (IMacro[])i.next();
@@ -139,29 +163,12 @@ public class IndexBasedCodeReaderFactory implements ICodeReaderFactory {
 		// No need for cache here
 		return null;
 	}
-
-	protected IIndexFragmentFile getCachedFile(String filename) throws CoreException {
-		IIndexFragmentFile file = (IIndexFragmentFile) fileCache.get(filename);
-		if (file == null) {
-			file = (IIndexFragmentFile) index.getFile(new Path(filename));
-			if (file != null) {
-				addFileToCache(filename, file);
-			}
-		}
-		return file;
-	}
 	
-	protected void addFileToCache(String filename, IIndexFile file) {
+	public FileInfo createFileInfo(String location) throws CoreException {
-		fileCache.put(filename, file);
+		return createInfo(location, null);
 	}
 
-	public IIndexFragmentFile createCachedFile(IWritableIndex index, String location) throws CoreException {
+	public FileInfo createFileInfo(ITranslationUnit tu) throws CoreException {
-		assert this.index == index;
+		return createInfo(tu.getLocation().toOSString(), null);
-		
-		IIndexFragmentFile file= getCachedFile(location);
-		if (file == null) {
-			file= index.addFile(location);
-		}
-		return file;
 	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/index/WritableCIndex.java

@@ -14,6 +14,7 @@ package org.eclipse.cdt.internal.core.index;
 import org.eclipse.cdt.core.dom.ast.IASTName;
 import org.eclipse.cdt.core.dom.ast.IASTPreprocessorMacroDefinition;
 import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IPath;
 
 public class WritableCIndex extends CIndex implements IWritableIndex {
 
@@ -32,13 +33,12 @@ public class WritableCIndex extends CIndex implements IWritableIndex {
 		return result;
 	}
 
-	public IIndexFragmentFile addFile(String fileLocation) throws CoreException {
+	public IIndexFragmentFile addFile(IPath fileLocation) throws CoreException {
 		IWritableIndexFragment frag= selectFragment(fileLocation);
-		
+		return frag.addFile(fileLocation.toOSString());
-		return frag.addFile(fileLocation);
 	}
 
-	private IWritableIndexFragment selectFragment(String fileLocation) {
+	private IWritableIndexFragment selectFragment(IPath fileLocation) {
 		// todo handling of multiple writable indices
 		assert fWritableFragments.length == 1;
 		return fWritableFragments[0];

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/ScannerUtility.java

@@ -22,10 +22,10 @@ import org.eclipse.cdt.core.parser.ISourceElementRequestor;
  */
 public class ScannerUtility {
 
-	static final char DOT    = '.';  //$NON-NLS-1$
+	static final char DOT    = '.';  
-	static final char SLASH  = '/';  //$NON-NLS-1$
+	static final char SLASH  = '/';
-	static final char BSLASH = '\\'; //$NON-NLS-1$
+	static final char BSLASH = '\\'; 
-	static final char QUOTE  = '\"'; //$NON-NLS-1$
+	static final char QUOTE  = '\"'; 
 
 	/**
 	 * This method is quick 1-pass path reconciler.

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/PDOM.java

@@ -64,7 +64,7 @@ public class PDOM extends PlatformObject implements IIndexFragment, IPDOM {
 
 	private Database db;
 
-	public static final int VERSION = 13;
+	public static final int VERSION = 14;
 	// 0 - the beginning of it all
 	// 1 - first change to kick off upgrades
 	// 2 - added file inclusions
@@ -149,22 +149,19 @@ public class PDOM extends PlatformObject implements IIndexFragment, IPDOM {
 		return fileIndex;
 	}
 
-	public PDOMFile getFile(String filename) throws CoreException {
+	public IIndexFragmentFile getFile(String path) throws CoreException {
-		Finder finder = new Finder(db, filename);
+		Finder finder = new Finder(db, path);
 		getFileIndex().accept(finder);
 		int record = finder.getRecord();
 		return record != 0 ? new PDOMFile(this, record) : null;
 	}
 
-	public IIndexFragmentFile getFile(IPath path) throws CoreException {
+	protected IIndexFragmentFile addFile(String path) throws CoreException {
-		return getFile(path.toOSString());
+		IIndexFragmentFile file = getFile(path);
-	}
-
-	protected IIndexFragmentFile addFile(String filename) throws CoreException {
-		PDOMFile file = getFile(filename);
 		if (file == null) {
-			file = new PDOMFile(this, filename);
+			PDOMFile pdomFile = new PDOMFile(this, path);
-			getFileIndex().insert(file.getRecord());
+			getFileIndex().insert(pdomFile.getRecord());
+			file= pdomFile;
 		}
 		return file;		
 	}
@@ -565,27 +562,7 @@ public class PDOM extends PlatformObject implements IIndexFragment, IPDOM {
 			return (PDOMFile) file;
 		}
 
-		return getFile(file.getLocation());
+		return (PDOMFile) getFile(file.getLocation());
-	}
-
-	public IIndexFragmentInclude[] findIncludes(IIndexFragmentFile file) throws CoreException {
-		PDOMFile pdomFile= adaptFile(file);
-		if (file != null) {
-			List result = new ArrayList();
-			for (PDOMInclude i= pdomFile.getFirstInclude(); i != null; i= i.getNextInIncludes()) {
-				result.add(i);
-			}
-			return (IIndexFragmentInclude[]) result.toArray(new PDOMInclude[result.size()]);
-		}
-		return new PDOMInclude[0];
-	}
-
-	public IIndexFragmentFile resolveInclude(IIndexFragmentInclude include) throws CoreException {
-		if (include.getFragment() == this && include instanceof PDOMInclude) {
-			PDOMInclude pdomInclude= (PDOMInclude) include;
-			return pdomInclude.getIncludes();
-		}
-		return getFile(include.getIncludesLocation());
 	}
 
 	public IPath getPath() {

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/PDOMIndexerJob.java

@@ -42,60 +42,76 @@ public class PDOMIndexerJob extends Job {
 
 	protected IStatus run(IProgressMonitor monitor) {
 		this.monitor = monitor;
-
 		long start = System.currentTimeMillis();
-		
+
 		String taskName = CCorePlugin.getResourceString("pdom.indexer.task"); //$NON-NLS-1$
 		monitor.beginTask(taskName, IProgressMonitor.UNKNOWN);
-		
-		while (!pdomManager.finishIndexerJob()) {
-			IPDOMIndexerTask nextTask= pdomManager.getNextTask();
-			synchronized (taskMutex) {
-				currentTask= nextTask;	// write to currentTask needs protection
-			}
 
-			try {
+		try {
-				currentTask.run(monitor);
+			do {
-			}
+				synchronized(taskMutex) {
-			catch (Exception e) {
+					currentTask= null;
-				CCorePlugin.log(e);
-			}
-			boolean cancelledByUser= false;
-			synchronized (taskMutex) {
-				currentTask= null;		// write to currentTask needs protection
-				if (cancelledByManager) {	
-					// TODO chance for confusion here is user cancels
-					// while project is getting deletes.
-					monitor.setCanceled(false);
-					cancelledByManager = false;
 					taskMutex.notify();
-				} 
+
-				else {
+					// user cancel, tell manager and return
-					cancelledByUser= monitor.isCanceled();
+					if (monitor.isCanceled()) {
+						pdomManager.cancelledJob(cancelledByManager);
+						return Status.CANCEL_STATUS;
+					}
+
+					// pick up new task
+					currentTask= pdomManager.getNextTask();
+				}
+
+				if (currentTask != null) {
+					try {
+						currentTask.run(monitor);
+					}
+					catch (Exception e) {
+						CCorePlugin.log(e);
+					}
 				}
 			}
-			if (cancelledByUser) {
+			while (currentTask != null);
-				pdomManager.cancelledByUser();
-				return Status.CANCEL_STATUS;
-			}
-		}
-		
-		String showTimings = Platform.getDebugOption(CCorePlugin.PLUGIN_ID
-				+ "/debug/pdomtimings"); //$NON-NLS-1$
-		if (showTimings != null && showTimings.equalsIgnoreCase("true")) //$NON-NLS-1$
-			System.out.println("PDOM Indexer Job Time: " + (System.currentTimeMillis() - start)); //$NON-NLS-1$
 
-		return Status.OK_STATUS;
+			String showTimings = Platform.getDebugOption(CCorePlugin.PLUGIN_ID
+					+ "/debug/pdomtimings"); //$NON-NLS-1$
+			if (showTimings != null && showTimings.equalsIgnoreCase("true")) //$NON-NLS-1$
+				System.out.println("PDOM Indexer Job Time: " + (System.currentTimeMillis() - start)); //$NON-NLS-1$
+
+			return Status.OK_STATUS;
+		}
+		catch (RuntimeException e) {
+			CCorePlugin.log(e);
+			pdomManager.cancelledJob(true);
+			synchronized (taskMutex) {
+				currentTask= null;
+				taskMutex.notify();
+			}
+			throw e;
+		}
+		catch (Error e) {
+			CCorePlugin.log(e);
+			pdomManager.cancelledJob(true);
+			synchronized (taskMutex) {
+				currentTask= null;
+				taskMutex.notify();
+			}
+			throw e;
+		}
 	}
 		
 	public void cancelJobs(IPDOMIndexer indexer) {
 		synchronized (taskMutex) {
-			if (currentTask != null && currentTask.getIndexer().equals(indexer)) {
+			if (currentTask != null && currentTask.getIndexer() == indexer) {
 				monitor.setCanceled(true);
 				cancelledByManager = true;
-				try {
+				while (currentTask != null && currentTask.getIndexer() == indexer) {
-					taskMutex.wait();
+					try {
-				} catch (InterruptedException e) {
+						taskMutex.wait();
+					} catch (InterruptedException e) {
+						return;
+					}
 				}
 			}
 		}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/PDOMManager.java

@@ -58,6 +58,7 @@ import org.eclipse.core.runtime.jobs.ISchedulingRule;
 import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.core.runtime.preferences.IEclipsePreferences;
 import org.eclipse.core.runtime.preferences.IPreferencesService;
+import org.eclipse.core.runtime.preferences.IScopeContext;
 import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.eclipse.core.runtime.preferences.IEclipsePreferences.IPreferenceChangeListener;
 import org.eclipse.core.runtime.preferences.IEclipsePreferences.PreferenceChangeEvent;
@@ -78,6 +79,7 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
 	private static final QualifiedName pdomProperty= new QualifiedName(CCorePlugin.PLUGIN_ID, "pdom"); //$NON-NLS-1$
 
 	private static final String INDEXER_ID_KEY = "indexerId"; //$NON-NLS-1$
+	private static final String INDEX_ALL_HEADERS = "indexAllHeaders"; //$NON-NLS-1$
 	private static final ISchedulingRule NOTIFICATION_SCHEDULING_RULE = new ISchedulingRule(){
 		public boolean contains(ISchedulingRule rule) {
 			return rule == this;
@@ -270,13 +272,42 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
     	job.setRule(project.getProject());
     	job.schedule(2000);
     }
-    
+
-	public IPDOMIndexer getIndexer(ICProject project) {
+    public void setIndexAllHeaders(final ICProject project, boolean val) {
+    	IEclipsePreferences prefs = new ProjectScope(project.getProject()).getNode(CCorePlugin.PLUGIN_ID);
+    	if (prefs == null)
+    		return; // TODO why would this be null?
+
+    	prefs.putBoolean(INDEX_ALL_HEADERS, val);
+    	Job job= new Job(Messages.PDOMManager_savePrefsJob) {
+        	protected IStatus run(IProgressMonitor monitor) {
+       	    	IEclipsePreferences prefs = new ProjectScope(project.getProject()).getNode(CCorePlugin.PLUGIN_ID);
+       	    	if (prefs != null) {
+       	    		try {
+       	    			prefs.flush();
+       	    		} catch (BackingStoreException e) {
+       	    		}
+       	    	}
+       	    	return Status.OK_STATUS;
+        	}
+    	};
+    	job.setSystem(true);
+    	job.setRule(project.getProject());
+    	job.schedule(2000);
+    }
+
+    public boolean getIndexAllHeaders(ICProject project) {
+    	IScopeContext[] scope= new IScopeContext[] {new ProjectScope(project.getProject()), new InstanceScope()};
+		return Platform.getPreferencesService().getBoolean(CCorePlugin.PLUGIN_ID, INDEX_ALL_HEADERS, true, scope); 
+    }
+
+    public IPDOMIndexer getIndexer(ICProject project) {
 		return getIndexer(project, true);
 	}
 	
 	public void onPreferenceChange(PreferenceChangeEvent event) {
-		if (event.getKey().equals(INDEXER_ID_KEY)) {
+		Object key= event.getKey();
+		if (key.equals(INDEXER_ID_KEY) || key.equals(INDEX_ALL_HEADERS)) {
 			Preferences node = event.getNode();
 			if (CCorePlugin.PLUGIN_ID.equals(node.name())) {
 				node= node.parent();
@@ -285,10 +316,7 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
 					ICProject cproject= CoreModel.getDefault().create(project);
 					if (cproject != null) {
 						try {
-							String newId= (String) event.getNewValue();
+							changeIndexer(cproject);
-							if (newId != null) {
-								changeIndexer(cproject, newId);
-							}
 						}
 						catch (Exception e) {
 							CCorePlugin.log(e);
@@ -299,17 +327,20 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
 		}
 	}
 
-	private void changeIndexer(ICProject cproject, String newid) throws CoreException {
+	private void changeIndexer(ICProject cproject) throws CoreException {
 		assert !Thread.holdsLock(fPDOMs);
 		IPDOMIndexer oldIndexer= null;
+		String newid= getIndexerId(cproject);
+		boolean allHeaders= getIndexAllHeaders(cproject);
+		
 		synchronized (fIndexerMutex) {
 			oldIndexer= getIndexer(cproject, false);
 			if (oldIndexer != null) {
-				if (oldIndexer.getID().equals(newid)) {
+				if (oldIndexer.getID().equals(newid) && oldIndexer.getIndexAllHeaders() == allHeaders) {
 					return;
 				}
 			}
-			createIndexer(cproject, newid, true);
+			createIndexer(cproject, newid, allHeaders, true);
 		}
 		
 		if (oldIndexer != null) {
@@ -339,7 +370,7 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
 
 			if (create) {
 				try {
-					return createIndexer(project, getIndexerId(project), false);
+					return createIndexer(project, getIndexerId(project), getIndexAllHeaders(project), false);
 				} catch (CoreException e) {
 					CCorePlugin.log(e);
 				}
@@ -348,7 +379,7 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
 		}
 	}
 		
-    private IPDOMIndexer createIndexer(ICProject project, String indexerId, boolean forceReindex) throws CoreException  {
+    private IPDOMIndexer createIndexer(ICProject project, String indexerId, boolean allHeaders, boolean forceReindex) throws CoreException  {
     	assert Thread.holdsLock(fIndexerMutex);
     	
     	PDOM pdom= (PDOM) getPDOM(project);
@@ -364,6 +395,7 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
     			if ("run".equals(element.getName())) { //$NON-NLS-1$
     				try {
 						indexer = (IPDOMIndexer)element.createExecutableExtension("class"); //$NON-NLS-1$
+						indexer.setIndexAllHeaders(allHeaders);
 					} catch (CoreException e) {
 						CCorePlugin.log(e);
 					} 
@@ -402,36 +434,45 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
     }
     
 	IPDOMIndexerTask getNextTask() {
+		boolean idle= false;
+		IPDOMIndexerTask result= null;
     	synchronized (taskQueueMutex) {
-    		currentTask= taskQueue.isEmpty() ? null : (IPDOMIndexerTask)taskQueue.removeFirst();
+    		if (taskQueue.isEmpty()) {
-    		return currentTask;
+    			currentTask= null;
+        		indexerJob= null;
+        		idle= true;
+    		}
+    		else {
+    			result= currentTask= (IPDOMIndexerTask)taskQueue.removeFirst();
+    		}
 		}
+    	if (idle) {
+    		notifyState(IndexerStateEvent.STATE_IDLE);
+    	}
+    	return result;
     }
     
-    void cancelledByUser() {
+    void cancelledJob(boolean byManager) {
-    	synchronized (taskQueueMutex) {
-    		taskQueue.clear();
-    		currentTask= null;
-    		indexerJob= null;
-		}
-		notifyState(IndexerStateEvent.STATE_IDLE);
-    }
-    
-    boolean finishIndexerJob() {
     	boolean idle= false;
     	synchronized (taskQueueMutex) {
     		currentTask= null;
-			if (taskQueue.isEmpty()) {
+    		if (!byManager) {
-				indexerJob = null;
+    			taskQueue.clear();
-				idle= true;
+    		}
-			} 
+    		idle= taskQueue.isEmpty();
+    		if (idle) {
+        		indexerJob= null;
+    		}
+    		else {
+    			indexerJob = new PDOMIndexerJob(this);
+    			indexerJob.schedule();
+    		}
     	}
     	if (idle) {
     		notifyState(IndexerStateEvent.STATE_IDLE);
     	}
-    	return idle;
     }
-    
+        
     private boolean isIndexerIdle() {
     	synchronized (taskQueueMutex) {
     		return currentTask == null && taskQueue.isEmpty();
@@ -482,6 +523,7 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
 		if (indexer != null) {
 			stopIndexer(indexer);
 		}
+    	unregisterPreferenceListener(project);
 	}
 
     public void deleteProject(ICProject project, IResourceDelta delta) {
@@ -490,6 +532,7 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
     	if (indexer != null) {
     		stopIndexer(indexer);
     	}
+    	unregisterPreferenceListener(project);
     }
 
 	private void stopIndexer(IPDOMIndexer indexer) {
@@ -507,11 +550,20 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
 				}
 			}
 		}
-		unregisterPreferenceListener(project);
+		PDOMIndexerJob jobToCancel= null;
 		synchronized (taskQueueMutex) {
-			if (indexerJob != null) {
+			for (Iterator iter = taskQueue.iterator(); iter.hasNext();) {
-				indexerJob.cancelJobs(indexer);
+				IPDOMIndexerTask task= (IPDOMIndexerTask) iter.next();
+				if (task.getIndexer() == indexer) {
+					iter.remove();
+				}
 			}
+			jobToCancel= indexerJob;
+		}
+		
+		if (jobToCancel != null) {
+			assert !Thread.holdsLock(taskQueueMutex);
+			jobToCancel.cancelJobs(indexer);
 		}
 	}    
 
@@ -532,6 +584,7 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
 	}
 
     private void notifyState(final int state) {
+    	assert !Thread.holdsLock(taskQueueMutex);
     	if (state == IndexerStateEvent.STATE_IDLE) {
     		synchronized(taskQueueMutex) {
     			taskQueueMutex.notifyAll();
@@ -612,25 +665,19 @@ public class PDOMManager implements IPDOMManager, IWritableIndexManager, IListen
 				if (monitor.isCanceled()) {
 					return false;
 				}
-				boolean hasTimedOut= false;
-				int wait= 1000;
-				if (waitMaxMillis >= 0) {
-					int rest= (int) (limit - System.currentTimeMillis());
-					if (rest < wait) {
-						if (rest <= 0) {
-							hasTimedOut= true;
-						}
-
-						wait= rest;
-					}
-				}
-				
 				synchronized(taskQueueMutex) {
 					if (isIndexerIdle()) {
 						return true;
 					}
-					if (hasTimedOut) {
+					int wait= 1000;
-						return false;
+					if (waitMaxMillis >= 0) {
+						int rest= (int) (limit - System.currentTimeMillis());
+						if (rest < wait) {
+							if (rest <= 0) {
+								return false;
+							}
+							wait= rest;
+						}
 					}
 
 					monitor.subTask(MessageFormat.format(Messages.PDOMManager_FilesToIndexSubtask, new Object[] {new Integer(getFilesToIndexCount())}));

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/Chunk.java

@@ -65,6 +65,14 @@ public class Chunk {
 	public int getInt(int offset) {
 		return buffer.getInt(offset % Database.CHUNK_SIZE);
 	}
+
+	public long getLong(int offset) {
+		return buffer.getLong(offset % Database.CHUNK_SIZE);
+	}
+
+	public void putLong(int offset, long value) {
+		buffer.putLong(offset % Database.CHUNK_SIZE, value);
+	}
 	
 	public void putChar(int offset, char value) {
 		buffer.putChar(offset % Database.CHUNK_SIZE, value);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/Database.java

@@ -271,6 +271,16 @@ public class Database {
 		return chunk.getInt(offset);
 	}
 
+	public void putLong(int offset, long value) throws CoreException {
+		Chunk chunk= getChunk(offset);
+		chunk.putLong(offset, value);
+	}
+	
+	public long getLong(int offset) throws CoreException {
+		Chunk chunk = getChunk(offset);
+		return chunk.getLong(offset);
+	}
+
 	public void putChar(int offset, char value) throws CoreException {
 		Chunk chunk = getChunk(offset);
 		chunk.putChar(offset, value);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMFile.java

@@ -42,8 +42,9 @@ public class PDOMFile implements IIndexFragmentFile {
 	private static final int FIRST_INCLUDED_BY = 8;
 	private static final int FIRST_MACRO = 12;
 	private static final int FILE_NAME = 16;
+	private static final int TIME_STAMP = 20;
 	
-	private static final int RECORD_SIZE = 20;
+	private static final int RECORD_SIZE = 28;
 	
 	public static class Comparator implements IBTreeComparator {
 		private Database db;
@@ -94,18 +95,12 @@ public class PDOMFile implements IIndexFragmentFile {
 		Database db = pdom.getDB();
 		record = db.malloc(RECORD_SIZE);
 		db.putInt(record + FILE_NAME, db.newString(filename).getRecord());
+		db.putLong(record + TIME_STAMP, 0);
 		setFirstName(null);
 		setFirstInclude(null);
 		setFirstIncludedBy(null);
 	}
 	
-	public void setFilename(String newName) throws CoreException {
-		Database db = pdom.getDB();
-		int oldRecord = db.getInt(record + FILE_NAME);
-		db.free(oldRecord);
-		db.putInt(record + FILE_NAME, db.newString(newName).getRecord());
-	}
-	
 	public int getRecord() {
 		return record;
 	}
@@ -125,6 +120,23 @@ public class PDOMFile implements IIndexFragmentFile {
 		return db.getString(db.getInt(record + FILE_NAME));
 	}
 	
+	public void setFilename(String newName) throws CoreException {
+		Database db = pdom.getDB();
+		int oldRecord = db.getInt(record + FILE_NAME);
+		db.free(oldRecord);
+		db.putInt(record + FILE_NAME, db.newString(newName).getRecord());
+	}
+
+	public long getTimestamp() throws CoreException {
+		Database db = pdom.getDB();
+		return db.getLong(record + TIME_STAMP);
+	}
+	
+	public void setTimestamp(long timestamp) throws CoreException {
+		Database db= pdom.getDB();
+		db.putLong(record + TIME_STAMP, timestamp);
+	}
+	
 	public PDOMName getFirstName() throws CoreException {
 		int namerec = pdom.getDB().getInt(record + FIRST_NAME);
 		return namerec != 0 ? new PDOMName(pdom, namerec) : null;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMName.java

@@ -14,6 +14,7 @@ package org.eclipse.cdt.internal.core.pdom.dom;
 import org.eclipse.cdt.core.CCorePlugin;
 import org.eclipse.cdt.core.dom.ast.IASTFileLocation;
 import org.eclipse.cdt.core.dom.ast.IASTName;
+import org.eclipse.cdt.core.index.IIndexFile;
 import org.eclipse.cdt.internal.core.index.IIndexFragment;
 import org.eclipse.cdt.internal.core.index.IIndexFragmentName;
 import org.eclipse.cdt.internal.core.index.IIndexProxyBinding;
@@ -142,7 +143,7 @@ public class PDOMName implements IIndexFragmentName, IASTFileLocation {
 		setNameField(BINDING_NEXT_OFFSET, name);
 	}
 	
-	public PDOMFile getFile() throws CoreException {
+	public IIndexFile getFile() throws CoreException {
 		int filerec = pdom.getDB().getInt(record + FILE_REC_OFFSET);
 		return filerec != 0 ? new PDOMFile(pdom, filerec) : null;
 	}
@@ -225,7 +226,7 @@ public class PDOMName implements IIndexFragmentName, IASTFileLocation {
 
 	public String getFileName() {
 		try {
-			PDOMFile file = getFile();
+			PDOMFile file = (PDOMFile) getFile();
 			return file != null ? file.getFileName().getString() : null;
 		} catch (CoreException e) {
 			CCorePlugin.log(e);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/PDOMIndexerTask.java

@@ -11,10 +11,19 @@
 
 package org.eclipse.cdt.internal.core.pdom.indexer;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
 
 import org.eclipse.cdt.core.CCorePlugin;
 import org.eclipse.cdt.core.dom.IPDOMIndexerTask;
+import org.eclipse.cdt.core.index.IIndex;
+import org.eclipse.cdt.core.index.IIndexFile;
+import org.eclipse.cdt.core.index.IIndexInclude;
+import org.eclipse.cdt.core.model.CoreModel;
+import org.eclipse.cdt.core.model.CoreModelUtil;
 import org.eclipse.cdt.core.model.ICElement;
 import org.eclipse.cdt.core.model.ICElementDelta;
 import org.eclipse.cdt.core.model.ICElementVisitor;
@@ -25,11 +34,14 @@ import org.eclipse.cdt.internal.core.index.IWritableIndex;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.Path;
 
 public abstract class PDOMIndexerTask implements IPDOMIndexerTask {
-
+	private static final Object NO_CONTEXT = new Object();
 	protected static final int MAX_ERRORS = 10;
+
 	protected int fErrorCount;
+	protected Map fContextMap= new HashMap();
 	
 	protected void processDelta(ICElementDelta delta, Collection added, Collection changed, Collection removed) throws CoreException {
 		int flags = delta.getFlags();
@@ -97,9 +109,9 @@ public abstract class PDOMIndexerTask implements IPDOMIndexerTask {
 		}
 	}
 	
-	protected void changeTU(ITranslationUnit tu) throws CoreException, InterruptedException {
+	protected void parseTU(ITranslationUnit tu) throws CoreException, InterruptedException {
 		try {
-			doChangeTU(tu);
+			doParseTU(tu);
 		}
 		catch (CoreException e) {
 			if (++fErrorCount <= MAX_ERRORS) {
@@ -111,7 +123,7 @@ public abstract class PDOMIndexerTask implements IPDOMIndexerTask {
 		}
 	}
 
-	abstract protected void doChangeTU(ITranslationUnit tu) throws CoreException, InterruptedException;
+	abstract protected void doParseTU(ITranslationUnit tu) throws CoreException, InterruptedException;
 	
 	protected void clearIndex(IWritableIndex index) throws InterruptedException, CoreException {
 		// reset error count
@@ -125,4 +137,50 @@ public abstract class PDOMIndexerTask implements IPDOMIndexerTask {
 			index.releaseWriteLock(0);
 		}
 	}
+	
+	protected boolean indexAllHeaders() {
+		return getIndexer().getIndexAllHeaders();
+	}
+	
+	protected ITranslationUnit findContext(IIndex index, String path) {
+		Object cachedContext= fContextMap.get(path);
+		if (cachedContext != null) {
+			return cachedContext == NO_CONTEXT ? null : (ITranslationUnit) cachedContext;
+		}
+		
+		ITranslationUnit context= null;
+		fContextMap.put(path, NO_CONTEXT); // prevent recursion
+		IIndexFile pdomFile;
+		try {
+			pdomFile = index.getFile(new Path(path));
+			if (pdomFile != null) {
+				ICProject project= getIndexer().getProject();
+				IIndexInclude[] includedBy = index.findIncludedBy(pdomFile, IIndex.DEPTH_ZERO);
+				ArrayList paths= new ArrayList(includedBy.length);
+				for (int i = 0; i < includedBy.length; i++) {
+					IIndexInclude include = includedBy[i];
+					String incfilename = include.getIncludedByLocation();
+					if (CoreModel.isValidSourceUnitName(project.getProject(), incfilename)) {
+						context= CoreModelUtil.findTranslationUnitForLocation(new Path(incfilename), project);
+						if (context != null) {
+							fContextMap.put(path, context);
+							return context;
+						}
+					}
+					paths.add(incfilename);
+				}
+				for (Iterator iter = paths.iterator(); iter.hasNext();) {
+					String nextLevel = (String) iter.next();
+					context= findContext(index, nextLevel);
+					if (context != null) {
+						fContextMap.put(path, context);
+						return context;
+					}
+				}
+			}
+		} catch (CoreException e) {
+			CCorePlugin.log(e);
+		}
+		return null;
+	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/fast/PDOMFastHandleDelta.java

@@ -27,7 +27,6 @@ class PDOMFastHandleDelta extends PDOMFastIndexerJob {
 
 	private List changed = new ArrayList();
 	private List removed = new ArrayList();
-	private volatile int fFilesToIndex= 0;
 
 	public PDOMFastHandleDelta(PDOMFastIndexer indexer, ICElementDelta delta) throws CoreException {
 		super(indexer);
@@ -37,18 +36,17 @@ class PDOMFastHandleDelta extends PDOMFastIndexerJob {
 
 	public void run(IProgressMonitor monitor) {
 		try {
-			setupIndexAndReaderFactory();
 			long start = System.currentTimeMillis();
-			Iterator i = changed.iterator();
+
-			while (i.hasNext()) {
+			setupIndexAndReaderFactory();
-				if (monitor.isCanceled())
+			registerTUsInReaderFactory(changed);
-					return;
+			
-				ITranslationUnit tu = (ITranslationUnit)i.next();
+			parseTUs(changed, monitor);
-				changeTU(tu);
+			if (monitor.isCanceled()) {
-				fFilesToIndex--;
+				return;
 			}
-						
+			
-			i = removed.iterator();
+			Iterator i= removed.iterator();
 			while (i.hasNext()) {
 				if (monitor.isCanceled())
 					return;
@@ -66,8 +64,4 @@ class PDOMFastHandleDelta extends PDOMFastIndexerJob {
 		} catch (InterruptedException e) {
 		}
 	}
-	
-	public int getFilesToIndexCount() {
-		return fFilesToIndex;
-	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/fast/PDOMFastIndexer.java

@@ -30,6 +30,8 @@ public class PDOMFastIndexer implements IPDOMIndexer {
 	public static final String ID = IPDOMManager.ID_FAST_INDEXER;
 	
 	protected ICProject project;
+
+	private boolean fIndexAllHeaders;
 	
 	public PDOMFastIndexer() {
 	}
@@ -56,4 +58,12 @@ public class PDOMFastIndexer implements IPDOMIndexer {
 	public String getID() {
 		return ID;
 	}
+
+	public void setIndexAllHeaders(boolean val) {
+		fIndexAllHeaders= val;
+	}
+
+	public boolean getIndexAllHeaders() {
+		return fIndexAllHeaders;
+	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/fast/PDOMFastIndexerJob.java

@@ -12,6 +12,14 @@
 
 package org.eclipse.cdt.internal.core.pdom.indexer.fast;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.eclipse.cdt.core.CCorePlugin;
 import org.eclipse.cdt.core.dom.IPDOMIndexer;
 import org.eclipse.cdt.core.dom.IPDOMIndexerTask;
@@ -29,9 +37,12 @@ import org.eclipse.cdt.internal.core.index.IIndexFragmentFile;
 import org.eclipse.cdt.internal.core.index.IWritableIndex;
 import org.eclipse.cdt.internal.core.index.IWritableIndexManager;
 import org.eclipse.cdt.internal.core.index.IndexBasedCodeReaderFactory;
+import org.eclipse.cdt.internal.core.index.IndexBasedCodeReaderFactory.FileInfo;
 import org.eclipse.cdt.internal.core.pdom.indexer.PDOMIndexerTask;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.Path;
 
 /**
  * @author Doug Schaefer
@@ -42,7 +53,8 @@ abstract class PDOMFastIndexerJob extends PDOMIndexerTask implements IPDOMIndexe
 	protected final PDOMFastIndexer indexer;
 	protected IWritableIndex index;
 	protected IndexBasedCodeReaderFactory codeReaderFactory;
-	
+	protected volatile int fFilesToIndex= 0;
+
 	public PDOMFastIndexerJob(PDOMFastIndexer indexer) throws CoreException {
 		this.indexer = indexer;
 	}
@@ -52,11 +64,19 @@ abstract class PDOMFastIndexerJob extends PDOMIndexerTask implements IPDOMIndexe
 		this.codeReaderFactory = new IndexBasedCodeReaderFactory(index);
 	}		
 
+	protected void registerTUsInReaderFactory(Collection tus) throws CoreException {
+		for (Iterator iter = tus.iterator(); iter.hasNext();) {
+			ITranslationUnit tu = (ITranslationUnit) iter.next();
+			FileInfo info= codeReaderFactory.createFileInfo(tu);
+			info.fNeedToIndex= true;
+		}
+	}
+	
 	public IPDOMIndexer getIndexer() {
 		return indexer;
 	}
 	
-	protected void doChangeTU(ITranslationUnit tu) throws CoreException, InterruptedException {
+	protected void doParseTU(ITranslationUnit tu) throws CoreException, InterruptedException {
 		IPath path = tu.getLocation();
 		if (path == null) {
 			return;
@@ -80,13 +100,8 @@ abstract class PDOMFastIndexerJob extends PDOMIndexerTask implements IPDOMIndexe
 			index.acquireWriteLock(1);
 			try {
 				// Clear the macros
-				codeReaderFactory.clearMacros();
+				codeReaderFactory.clearMacroAttachements();
 				
-				// Remove the old symbols in the tu
-				IIndexFragmentFile file= (IIndexFragmentFile) index.getFile(path);
-				if (file != null)
-					index.clearFile(file);
-
 				// Add the new symbols
 				addSymbols(ast);
 			} finally {
@@ -100,37 +115,29 @@ abstract class PDOMFastIndexerJob extends PDOMIndexerTask implements IPDOMIndexe
 
 	protected void addSymbols(IASTTranslationUnit ast) throws InterruptedException, CoreException {
 		// Add in the includes
+		final LinkedHashMap symbolMap= new LinkedHashMap(); // makes bugs reproducible
+		
+		// includes
 		IASTPreprocessorIncludeStatement[] includes = ast.getIncludeDirectives();
 		for (int i = 0; i < includes.length; ++i) {
 			IASTPreprocessorIncludeStatement include = includes[i];
-			
 			IASTFileLocation sourceLoc = include.getFileLocation();
-			String sourcePath
+			String path= sourceLoc != null ? sourceLoc.getFileName() : ast.getFilePath(); // command-line includes
-				= sourceLoc != null
+			addToMap(symbolMap, 0, path, include);
-				? sourceLoc.getFileName()
-				: ast.getFilePath(); // command-line includes
-				
-			IIndexFragmentFile sourceFile = codeReaderFactory.createCachedFile(index, sourcePath);
-			String destPath = include.getPath();
-			IIndexFragmentFile destFile = codeReaderFactory.createCachedFile(index, destPath);
-			index.addInclude(sourceFile, destFile);
 		}
 	
-		// Add in the macros
+		// macros
 		IASTPreprocessorMacroDefinition[] macros = ast.getMacroDefinitions();
 		for (int i = 0; i < macros.length; ++i) {
 			IASTPreprocessorMacroDefinition macro = macros[i];
-			
 			IASTFileLocation sourceLoc = macro.getFileLocation();
-			if (sourceLoc == null)
+			if (sourceLoc != null) { // skip built-ins and command line macros
-				continue; // skip built-ins and command line macros
+				String path = sourceLoc.getFileName();
-				
+				addToMap(symbolMap, 1, path, macro);
-			String filename = sourceLoc.getFileName();
+			}
-			IIndexFragmentFile sourceFile = codeReaderFactory.createCachedFile(index, filename);
-			index.addMacro(sourceFile, macro);
 		}
 			
-		// Add in the names
+		// names
 		ast.accept(new ASTVisitor() {
 			{
 				shouldVisitNames = true;
@@ -139,8 +146,9 @@ abstract class PDOMFastIndexerJob extends PDOMIndexerTask implements IPDOMIndexe
 			public int visit(IASTName name) {
 				try {
 					IASTFileLocation nameLoc = name.getFileLocation();
-					if (nameLoc != null)
+					if (nameLoc != null) {
-						index.addName(codeReaderFactory.createCachedFile(index, nameLoc.getFileName()), name);
+						addToMap(symbolMap, 2, nameLoc.getFileName(), name);
+					}
 					return PROCESS_CONTINUE;
 				} catch (Throwable e) {
 					CCorePlugin.log(e);
@@ -148,6 +156,136 @@ abstract class PDOMFastIndexerJob extends PDOMIndexerTask implements IPDOMIndexe
 				}
 			}
 		});
-	
+
+		for (Iterator iter = symbolMap.entrySet().iterator(); iter.hasNext();) {
+			Map.Entry entry = (Map.Entry) iter.next();
+			String path= (String) entry.getKey();
+			FileInfo info= codeReaderFactory.createFileInfo(path);
+			if (!info.fNeedToIndex && info.fFile != null) {
+				iter.remove();
+			}
+			else {
+				// resolve the names
+				ArrayList names= ((ArrayList[]) entry.getValue())[2];
+				for (int i=0; i<names.size(); i++) {
+					((IASTName) names.get(i)).resolveBinding();
+				}
+			}				
+		}
+
+		for (Iterator iter = symbolMap.entrySet().iterator(); iter.hasNext();) {
+			Map.Entry entry = (Map.Entry) iter.next();
+			String path= (String) entry.getKey();
+			FileInfo info= codeReaderFactory.createFileInfo(path);
+			info.fNeedToIndex= false;
+			addToIndex(path, info, (ArrayList[]) entry.getValue());
+		}
+	}
+
+	private void addToMap(HashMap map, int idx, String path, Object thing) {
+		List[] lists= (List[]) map.get(path);
+		if (lists == null) {
+			lists= new ArrayList[]{new ArrayList(), new ArrayList(), new ArrayList()};
+			map.put(path, lists);
+		}
+		lists[idx].add(thing);
 	}		
+
+	private void addToIndex(String location, FileInfo info, ArrayList[] lists) throws CoreException {
+		// Remove the old symbols in the tu
+		Path path= new Path(location);
+		IIndexFragmentFile file= (IIndexFragmentFile) info.fFile;
+		if (file != null) {
+			index.clearFile(file);
+		}
+		else {
+			file= index.addFile(path);
+			info.fFile= file;
+		}
+		file.setTimestamp(path.toFile().lastModified());
+
+		// includes
+		ArrayList list= lists[0];
+		for (int i = 0; i < list.size(); i++) {
+			IASTPreprocessorIncludeStatement include= (IASTPreprocessorIncludeStatement) list.get(i);
+			IIndexFragmentFile destFile= createIndexFile(include.getPath());
+			index.addInclude(file, destFile);
+		}
+
+		// macros
+		list= lists[1];
+		for (int i = 0; i < list.size(); i++) {
+			index.addMacro(file, (IASTPreprocessorMacroDefinition) list.get(i));
+		}
+
+		// symbols
+		list= lists[2];
+		for (int i = 0; i < list.size(); i++) {
+			index.addName(file, (IASTName) list.get(i));
+		}		
+	}
+
+	private IIndexFragmentFile createIndexFile(String path) throws CoreException {
+		FileInfo info= codeReaderFactory.createFileInfo(path);
+		if (info.fFile == null) {
+			info.fFile= index.addFile(new Path(path));
+		}
+		return (IIndexFragmentFile) info.fFile;
+	}
+	
+	protected void parseTUs(List translationUnits, IProgressMonitor monitor) throws CoreException, InterruptedException {
+		// sources first
+		Iterator i = translationUnits.iterator();
+		while (i.hasNext()) {
+			if (monitor.isCanceled()) 
+				return;
+			ITranslationUnit tu = (ITranslationUnit)i.next();
+			if (tu.isSourceUnit()) {
+				parseTU(tu);
+				i.remove();
+				fFilesToIndex--;
+			}
+		}
+
+		// headers with context
+		i = translationUnits.iterator();
+		while (i.hasNext()) {
+			if (monitor.isCanceled())
+				return;
+			ITranslationUnit tu = (ITranslationUnit)i.next();
+			FileInfo info= codeReaderFactory.createFileInfo(tu);
+			if (!info.fNeedToIndex) {
+				i.remove();
+				fFilesToIndex--;
+			}
+			else if (info.fFile != null) {
+				ITranslationUnit context= findContext(index, info.fFile.getLocation());
+				if (context != null) {
+					parseTU(context);
+				}
+			}
+		}
+
+		// headers without context
+		if (indexAllHeaders()) {
+			i = translationUnits.iterator();
+			while (i.hasNext()) {
+				if (monitor.isCanceled())
+					return;
+				ITranslationUnit tu = (ITranslationUnit)i.next();
+				FileInfo info= codeReaderFactory.createFileInfo(tu);
+				if (!info.fNeedToIndex) {
+					i.remove();
+				}
+				else {
+					parseTU(tu);
+				}
+				fFilesToIndex--;
+			}
+		}
+	}
+	
+	final public int getFilesToIndexCount() {
+		return fFilesToIndex;
+	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/fast/PDOMFastReindex.java

@@ -13,10 +13,8 @@
 package org.eclipse.cdt.internal.core.pdom.indexer.fast;
 
 import java.util.ArrayList;
-import java.util.Iterator;
 
 import org.eclipse.cdt.core.CCorePlugin;
-import org.eclipse.cdt.core.model.ITranslationUnit;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.Platform;
@@ -26,8 +24,6 @@ import org.eclipse.core.runtime.Platform;
  *
  */
 class PDOMFastReindex extends PDOMFastIndexerJob {
-
-	private volatile int fFilesToIndex= 0;
 	private ArrayList fTUs= new ArrayList();
 	
 	public PDOMFastReindex(PDOMFastIndexer indexer) throws CoreException {
@@ -40,14 +36,12 @@ class PDOMFastReindex extends PDOMFastIndexerJob {
 		try {
 			long start = System.currentTimeMillis();
 			setupIndexAndReaderFactory();
+			registerTUsInReaderFactory(fTUs);
+			
 			clearIndex(index);
 			fFilesToIndex--;
 
-			for (Iterator iter = fTUs.iterator(); iter.hasNext();) {
+			parseTUs(fTUs, monitor);
-				ITranslationUnit tu = (ITranslationUnit) iter.next();
-				changeTU(tu);
-				fFilesToIndex--;
-			}
 
 			assert fFilesToIndex == 0;
 			String showTimings = Platform.getDebugOption(CCorePlugin.PLUGIN_ID
@@ -61,9 +55,4 @@ class PDOMFastReindex extends PDOMFastIndexerJob {
 		} catch (InterruptedException e) {
 		}
 	}
-
-	public int getFilesToIndexCount() {
-		return fFilesToIndex;
-	}
-
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/full/PDOMFullHandleDelta.java

@@ -13,25 +13,14 @@
 package org.eclipse.cdt.internal.core.pdom.indexer.full;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 
 import org.eclipse.cdt.core.CCorePlugin;
-import org.eclipse.cdt.core.index.IIndex;
-import org.eclipse.cdt.core.index.IIndexFile;
-import org.eclipse.cdt.core.index.IIndexInclude;
-import org.eclipse.cdt.core.model.CoreModel;
 import org.eclipse.cdt.core.model.ICElementDelta;
 import org.eclipse.cdt.core.model.ITranslationUnit;
-import org.eclipse.core.resources.IFile;
-import org.eclipse.core.resources.IProject;
-import org.eclipse.core.resources.ResourcesPlugin;
 import org.eclipse.core.runtime.CoreException;
-import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
-import org.eclipse.core.runtime.Path;
 import org.eclipse.core.runtime.Platform;
 
 /**
@@ -40,107 +29,42 @@ import org.eclipse.core.runtime.Platform;
  */
 class PDOMFullHandleDelta extends PDOMFullIndexerJob {
 
-	// Map of filename, TU of files that need to be parsed.
-	private Map changedMap = new HashMap();
 	private List changed = new ArrayList();
-	private List added = new ArrayList();
 	private List removed = new ArrayList();
-	private volatile int fFilesToIndex= 0;
+
-	
 	public PDOMFullHandleDelta(PDOMFullIndexer indexer, ICElementDelta delta) throws CoreException {
 		super(indexer);
-		processDelta(delta, added, changed, removed);
+		processDelta(delta, changed, changed, removed);
+		fFilesToIndex= changed.size() + removed.size();
 	}
 
 	public void run(IProgressMonitor monitor) {
-		setupIndexAndReaderFactory();
 		try {
 			long start = System.currentTimeMillis();
+			setupIndexAndReaderFactory();
+			registerTUsInReaderFactory(changed);
 					
-			int count = changed.size() + added.size() + removed.size();
+			parseTUs(changed, monitor);
-
+			if (monitor.isCanceled()) {
-			for (Iterator iter = changed.iterator(); iter.hasNext();) {
+				return;
-				ITranslationUnit tu = (ITranslationUnit) iter.next();
+			}
-				processTranslationUnit(tu);
+			
+			Iterator i= removed.iterator();
+			while (i.hasNext()) {
+				if (monitor.isCanceled())
+					return;
+				ITranslationUnit tu = (ITranslationUnit)i.next();
+				removeTU(index, tu);
 				fFilesToIndex--;
 			}
-			changed.clear();
-			
-			if (count > 0) {
-				Iterator i = changedMap.values().iterator();
-				while (i.hasNext()) {
-					if (monitor.isCanceled())
-						return;
-					ITranslationUnit tu = (ITranslationUnit)i.next();
-					changeTU(tu);
-					fFilesToIndex--;
-				}
 				
-				i = added.iterator();
+			String showTimings = Platform.getDebugOption(CCorePlugin.PLUGIN_ID
-				while (i.hasNext()) {
-					if (monitor.isCanceled())
-						return;
-					ITranslationUnit tu = (ITranslationUnit)i.next();
-					changeTU(tu);
-					fFilesToIndex--;
-				}
-				
-				i = removed.iterator();
-				while (i.hasNext()) {
-					if (monitor.isCanceled())
-						return;
-					ITranslationUnit tu = (ITranslationUnit)i.next();
-					removeTU(index, tu);
-					fFilesToIndex--;
-				}
-				
-				String showTimings = Platform.getDebugOption(CCorePlugin.PLUGIN_ID
 						+ "/debug/pdomtimings"); //$NON-NLS-1$
 				if (showTimings != null && showTimings.equalsIgnoreCase("true")) //$NON-NLS-1$
 					System.out.println("PDOM Full Delta Time: " + (System.currentTimeMillis() - start)); //$NON-NLS-1$
-			}
 		} catch (CoreException e) {
 			CCorePlugin.log(e);
 		} catch (InterruptedException e) {
 		}
 	}
-
-	
-	protected void processTranslationUnit(ITranslationUnit tu) throws CoreException {
-		IPath path = tu.getUnderlyingResource().getLocation();
-		IIndexFile pdomFile= index.getFile(path);
-		boolean found = false;
-		if (pdomFile != null) {
-			// Look for all source units in the included list,
-			// If none, then add the header
-			IIndexInclude[] includedBy = index.findIncludedBy(pdomFile, IIndex.DEPTH_INFINITE); 
-			if (includedBy.length > 0) {
-				IProject project = tu.getCProject().getProject();
-				for (int i = 0; i < includedBy.length; ++i) {
-					String incfilename = includedBy[i].getIncludedByLocation();
-					if (CoreModel.isValidSourceUnitName(project, incfilename)) {
-						if (changedMap.get(incfilename) == null) {
-							IFile[] rfiles = ResourcesPlugin.getWorkspace().getRoot().findFilesForLocation(new Path(incfilename));
-							for (int j = 0; j < rfiles.length; ++j) {
-								if (rfiles[j].getProject().equals(project)) {
-									ITranslationUnit inctu = (ITranslationUnit)CoreModel.getDefault().create(rfiles[j]);
-									changedMap.put(incfilename, inctu);
-									found = true;
-									fFilesToIndex++;
-								}
-							}
-						}
-					}
-				}
-			}
-		}
-		if (!found) {
-			changedMap.put(path.toOSString(), tu);
-			fFilesToIndex++;
-		}
-	}
-
-	public int getFilesToIndexCount() {
-		return fFilesToIndex;
-	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/full/PDOMFullIndexer.java

@@ -28,7 +28,9 @@ import org.eclipse.core.runtime.CoreException;
 public class PDOMFullIndexer implements IPDOMIndexer {
 	public static final String ID = IPDOMManager.ID_FULL_INDEXER;
 	
+	private boolean fIndexAllHeaders= true;
 	private ICProject project;
+
 	
 	public ICProject getProject() {
 		return project;
@@ -39,7 +41,10 @@ public class PDOMFullIndexer implements IPDOMIndexer {
 	}
 	
 	public void handleDelta(ICElementDelta delta) throws CoreException {
-		CCoreInternals.getPDOMManager().enqueue(new PDOMFullHandleDelta(this, delta));
+		PDOMFullHandleDelta task = new PDOMFullHandleDelta(this, delta);
+		if (task.getFilesToIndexCount() > 0) {
+			CCoreInternals.getPDOMManager().enqueue(task);
+		}
 	}
 
 	public void reindex() throws CoreException {
@@ -49,4 +54,12 @@ public class PDOMFullIndexer implements IPDOMIndexer {
 	public String getID() {
 		return ID;
 	}
+
+	public void setIndexAllHeaders(boolean val) {
+		fIndexAllHeaders= val;
+	}
+
+	public boolean getIndexAllHeaders() {
+		return fIndexAllHeaders;
+	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/full/PDOMFullIndexerJob.java

@@ -12,6 +12,16 @@
 
 package org.eclipse.cdt.internal.core.pdom.indexer.full;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.eclipse.cdt.core.CCorePlugin;
 import org.eclipse.cdt.core.dom.IPDOMIndexer;
 import org.eclipse.cdt.core.dom.IPDOMIndexerTask;
@@ -28,6 +38,7 @@ import org.eclipse.cdt.internal.core.index.IWritableIndexManager;
 import org.eclipse.cdt.internal.core.pdom.indexer.PDOMIndexerTask;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.Path;
 
 /**
@@ -37,22 +48,83 @@ import org.eclipse.core.runtime.Path;
 abstract class PDOMFullIndexerJob extends PDOMIndexerTask implements IPDOMIndexerTask {
 
 	protected final PDOMFullIndexer indexer;
-	protected final IWritableIndex index;
+	protected IWritableIndex index= null;
-		
+	private Set filePathsToParse= null;
+	protected volatile int fFilesToIndex= 0;
+
 	public PDOMFullIndexerJob(PDOMFullIndexer indexer) throws CoreException {
 		this.indexer = indexer;
-		this.index = ((IWritableIndexManager) CCorePlugin.getIndexManager()).getWritableIndex(indexer.getProject());
 	}
 
 	public IPDOMIndexer getIndexer() {
 		return indexer;
 	}
 	
-	protected void setupIndexAndReaderFactory() {
+	protected void setupIndexAndReaderFactory() throws CoreException {
-		// mstodo delay setting up index to here.
+		this.index = ((IWritableIndexManager) CCorePlugin.getIndexManager()).getWritableIndex(indexer.getProject());
 	}
 	
-	protected void doChangeTU(ITranslationUnit tu) throws CoreException, InterruptedException {
+	protected void registerTUsInReaderFactory(Collection tus) throws CoreException {
+		filePathsToParse= new HashSet();
+		
+		for (Iterator iter = tus.iterator(); iter.hasNext();) {
+			ITranslationUnit tu = (ITranslationUnit) iter.next();
+			filePathsToParse.add(tu.getLocation().toOSString());
+		}
+	}
+	
+	protected void parseTUs(Collection translationUnits, IProgressMonitor monitor) throws CoreException, InterruptedException {
+		// sources first
+		Iterator i = translationUnits.iterator();
+		while (i.hasNext()) {
+			if (monitor.isCanceled()) 
+				return;
+			ITranslationUnit tu = (ITranslationUnit)i.next();
+			String path = tu.getLocation().toOSString();
+			if (!filePathsToParse.contains(path)) {
+				i.remove();
+			}
+			else if (tu.isSourceUnit()) {
+				parseTU(tu);
+				i.remove();
+			}
+		}
+
+		// headers with context
+		i = translationUnits.iterator();
+		while (i.hasNext()) {
+			if (monitor.isCanceled())
+				return;
+			ITranslationUnit tu = (ITranslationUnit)i.next();
+			String path = tu.getLocation().toOSString();
+			if (!filePathsToParse.contains(path)) {
+				i.remove();
+			}
+			else {
+				ITranslationUnit context= findContext(index, path);
+				if (context != null) {
+					parseTU(context);
+				}
+			}
+		}
+
+		// headers without context
+		if (indexAllHeaders()) {
+			i = translationUnits.iterator();
+			while (i.hasNext()) {
+				ITranslationUnit tu = (ITranslationUnit)i.next();
+				String path = tu.getLocation().toOSString();
+				if (!filePathsToParse.contains(path)) {
+					i.remove();
+				}
+				else {
+					parseTU(tu);
+				}
+			}
+		}
+	}
+
+	protected void doParseTU(ITranslationUnit tu) throws CoreException, InterruptedException {
 		IPath path = tu.getLocation();
 		if (path == null) {
 			return;
@@ -60,7 +132,7 @@ abstract class PDOMFullIndexerJob extends PDOMIndexerTask implements IPDOMIndexe
 		IASTTranslationUnit ast= tu.getAST(null, ITranslationUnit.AST_SKIP_IF_NO_BUILD_INFO);
 		if (ast == null)
 			return;
-		
+		System.out.println(path.toOSString());
 		index.acquireWriteLock(0);
 		
 		try {
@@ -85,39 +157,31 @@ abstract class PDOMFullIndexerJob extends PDOMIndexerTask implements IPDOMIndexe
 		}
 	}
 	
-	protected void addSymbols(IASTTranslationUnit ast) throws CoreException {
+	protected void addSymbols(IASTTranslationUnit ast) throws InterruptedException, CoreException {
 		// Add in the includes
+		final LinkedHashMap symbolMap= new LinkedHashMap(); // makes bugs reproducible
+		
+		// includes
 		IASTPreprocessorIncludeStatement[] includes = ast.getIncludeDirectives();
 		for (int i = 0; i < includes.length; ++i) {
 			IASTPreprocessorIncludeStatement include = includes[i];
-			
 			IASTFileLocation sourceLoc = include.getFileLocation();
-			String sourcePath
+			String path= sourceLoc != null ? sourceLoc.getFileName() : ast.getFilePath(); // command-line includes
-				= sourceLoc != null
+			addToMap(symbolMap, 0, path, include);
-				? sourceLoc.getFileName()
-				: ast.getFilePath(); // command-line includes
-				
-			IIndexFragmentFile sourceFile = index.addFile(sourcePath);
-			String destPath = include.getPath();
-			IIndexFragmentFile destFile = index.addFile(destPath);
-			index.addInclude(sourceFile, destFile);
 		}
-					
+	
-		// Add in the macros
+		// macros
 		IASTPreprocessorMacroDefinition[] macros = ast.getMacroDefinitions();
 		for (int i = 0; i < macros.length; ++i) {
 			IASTPreprocessorMacroDefinition macro = macros[i];
-			
 			IASTFileLocation sourceLoc = macro.getFileLocation();
-			if (sourceLoc == null)
+			if (sourceLoc != null) { // skip built-ins and command line macros
-				continue; // skip built-ins and command line macros
+				String path = sourceLoc.getFileName();
-				
+				addToMap(symbolMap, 1, path, macro);
-			String filename = sourceLoc.getFileName();
+			}
-			IIndexFragmentFile sourceFile = index.addFile(filename);
-			index.addMacro(sourceFile, macro);
 		}
-					
+			
-		// Add in the names
+		// names
 		ast.accept(new ASTVisitor() {
 			{
 				shouldVisitNames = true;
@@ -126,8 +190,9 @@ abstract class PDOMFullIndexerJob extends PDOMIndexerTask implements IPDOMIndexe
 			public int visit(IASTName name) {
 				try {
 					IASTFileLocation nameLoc = name.getFileLocation();
-					if (nameLoc != null)
+					if (nameLoc != null) {
-						index.addName(index.addFile(nameLoc.getFileName()), name);
+						addToMap(symbolMap, 2, nameLoc.getFileName(), name);
+					}
 					return PROCESS_CONTINUE;
 				} catch (Throwable e) {
 					CCorePlugin.log(e);
@@ -135,5 +200,72 @@ abstract class PDOMFullIndexerJob extends PDOMIndexerTask implements IPDOMIndexe
 				}
 			}
 		});
+
+		for (Iterator iter = symbolMap.values().iterator(); iter.hasNext();) {
+			// resolve the names
+			ArrayList names= ((ArrayList[]) iter.next())[2];
+			for (int i=0; i<names.size(); i++) {
+				((IASTName) names.get(i)).resolveBinding();
+			}
+		}
+
+		for (Iterator iter = symbolMap.entrySet().iterator(); iter.hasNext();) {
+			Map.Entry entry = (Map.Entry) iter.next();
+			String path= (String) entry.getKey();
+			addToIndex(path, (ArrayList[]) entry.getValue());
+		}
+	}
+
+	private void addToMap(HashMap map, int idx, String path, Object thing) {
+		if (filePathsToParse.contains(path)) {
+			List[] lists= (List[]) map.get(path);
+			if (lists == null) {
+				lists= new ArrayList[]{new ArrayList(), new ArrayList(), new ArrayList()};
+				map.put(path, lists);
+			}
+			lists[idx].add(thing);
+		}
+	}		
+
+	private void addToIndex(String location, ArrayList[] lists) throws CoreException {
+		if (!filePathsToParse.remove(location)) {
+			return;
+		}
+		fFilesToIndex--;
+
+		// Remove the old symbols in the tu
+		Path path= new Path(location);
+		IIndexFragmentFile file= (IIndexFragmentFile) index.getFile(new Path(location));
+		if (file != null) {
+			index.clearFile(file);
+		}
+		else {
+			file= index.addFile(path);
+		}
+		file.setTimestamp(path.toFile().lastModified());
+		
+		// includes
+		ArrayList list= lists[0];
+		for (int i = 0; i < list.size(); i++) {
+			IASTPreprocessorIncludeStatement include= (IASTPreprocessorIncludeStatement) list.get(i);
+			IIndexFragmentFile destFile= index.addFile(new Path(include.getPath()));
+			index.addInclude(file, destFile);
+		}
+
+		// macros
+		list= lists[1];
+		for (int i = 0; i < list.size(); i++) {
+			index.addMacro(file, (IASTPreprocessorMacroDefinition) list.get(i));
+		}
+
+		// symbols
+		list= lists[2];
+		for (int i = 0; i < list.size(); i++) {
+			index.addName(file, (IASTName) list.get(i));
+		}		
+	}
+	
+	final public int getFilesToIndexCount() {
+		return fFilesToIndex;
 	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/full/PDOMFullReindex.java

@@ -13,12 +13,9 @@
 package org.eclipse.cdt.internal.core.pdom.indexer.full;
 
 import java.util.ArrayList;
-import java.util.Iterator;
 
 import org.eclipse.cdt.core.CCorePlugin;
-import org.eclipse.cdt.core.model.ITranslationUnit;
 import org.eclipse.core.runtime.CoreException;
-import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.Status;
@@ -29,46 +26,27 @@ import org.eclipse.core.runtime.Status;
  */
 class PDOMFullReindex extends PDOMFullIndexerJob {
 
-	private volatile int fFilesToIndex= 0;
 	private ArrayList fTUs= new ArrayList();
-	private ArrayList fHeaders= new ArrayList();
 
 	public PDOMFullReindex(PDOMFullIndexer indexer) throws CoreException {
 		super(indexer);
-		collectSources(indexer.getProject(), fTUs, fHeaders);
+		collectSources(indexer.getProject(), fTUs, fTUs);
-		fFilesToIndex= fTUs.size()+fHeaders.size() + 1;
+		fFilesToIndex= fTUs.size() + 1;
 	}
 
 	public void run(final IProgressMonitor monitor) {
 		try {
+			System.out.println(this);
 			long start = System.currentTimeMillis();			
+
 			setupIndexAndReaderFactory();
+			registerTUsInReaderFactory(fTUs);
+			
 			clearIndex(index);
 			fFilesToIndex--;
 			
-			for (Iterator iter = fTUs.iterator(); iter.hasNext();) {
+			parseTUs(fTUs, monitor);
-				ITranslationUnit tu = (ITranslationUnit) iter.next();
-				if (monitor.isCanceled())
-					return;
-				changeTU(tu);
-				fFilesToIndex--;
-			}
-			
-			for (Iterator iter = fHeaders.iterator(); iter.hasNext();) {
-				ITranslationUnit tu = (ITranslationUnit) iter.next();
-				if (monitor.isCanceled())
-					return;
-			
-				IPath fileLocation = tu.getLocation();
-				if ( fileLocation != null ) {
-					if (index.getFile(fileLocation) == null) {
-						changeTU(tu);
-					}
-					fFilesToIndex--;
-				}
-			}
 
-			assert fFilesToIndex==0;
 			String showTimings = Platform.getDebugOption(CCorePlugin.PLUGIN_ID
 					+ "/debug/pdomtimings"); //$NON-NLS-1$
 			if (showTimings != null && showTimings.equalsIgnoreCase("true")) //$NON-NLS-1$
@@ -80,9 +58,4 @@ class PDOMFullReindex extends PDOMFullIndexerJob {
 		} catch (InterruptedException e) {
 		}
 	}
-
-	public int getFilesToIndexCount() {
-		return fFilesToIndex;
-	}
-
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/nulli/PDOMNullIndexer.java

@@ -34,6 +34,8 @@ public class PDOMNullIndexer implements IPDOMIndexer {
 	public static final String ID = IPDOMManager.ID_NO_INDEXER;
 	
 	private ICProject project;
+
+	private boolean fIndexAllHeaders;
 	
 	public ICProject getProject() {
 		return project;
@@ -81,4 +83,12 @@ public class PDOMNullIndexer implements IPDOMIndexer {
 	public String getID() {
 		return ID;
 	}
+
+	public boolean getIndexAllHeaders() {
+		return fIndexAllHeaders;
+	}
+
+	public void setIndexAllHeaders(boolean value) {
+		fIndexAllHeaders= value;
+	}
 }