Bug 422681: Allow references to bindings in other linkages

The existing PDOMBinding and PDOMName implementations do not allow
references between linkages.  This feature is needed so that the new Qt
linkage can reference elements in the C++ linkage.  It will also allow
the C++ implementation for extern "C" to be cleaned up (see
PDOM.getCrossLanguageBindings).

Prior to this change, a PDOMBinding held three lists of names.  One for
each of declarations, definitions, and references.  This change adds a
fourth list for external references.  External references are stored as
a linked list of nodes.  Each node holds:

    - The linkage id.
    - A pointer to the next node (or 0 for end-of-list).
    - A pointer to the record of first name in the list.

The linkage id is held separately because a PDOMName does not have any
field of it's own for linkage.  By grouping elements in this way, we can
reuse most of the existing list-related code.

External references are accessed through a new PDOMIterator class.  This
is needed so that we can advance to the next linkage node when we get to
the end of one node's list of names.

This also adds a unit test for the new API.

Change-Id: Ie2b14848db7409905beda0cec752080d5f42eec8
Signed-off-by: Andrew Eidsness <eclipse@jfront.com>
Reviewed-on: https://git.eclipse.org/r/18979
Tested-by: Hudson CI
Reviewed-by: Doug Schaefer <dschaefer@qnx.com>
IP-Clean: Doug Schaefer <dschaefer@qnx.com>

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/pdom/tests/PDOMCPPBugsTest.java

@@ -12,6 +12,7 @@
 package org.eclipse.cdt.internal.pdom.tests;
 
 import java.io.File;
+import java.util.regex.Pattern;
 
 import junit.framework.Test;
 
@@ -74,6 +75,7 @@ public class PDOMCPPBugsTest extends BaseTestCase {
 		pdom.acquireWriteLock(0);
 		try {
 			WritablePDOM wpdom = (WritablePDOM) pdom;
+			IIndexBinding[] b = wpdom.findBindings(Pattern.compile(".+"), false, IndexFilter.ALL, null);
 			wpdom.setProperty("a", "b");
 			assertEquals("b", wpdom.getProperty("a"));
 			wpdom.setProperty("c", "d");

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/pdom/tests/PDOMNameTests.java

@@ -0,0 +1,105 @@
+package org.eclipse.cdt.internal.pdom.tests;
+
+import junit.framework.Test;
+
+import org.eclipse.cdt.core.CCorePlugin;
+import org.eclipse.cdt.core.dom.IPDOMManager;
+import org.eclipse.cdt.core.index.IIndexBinding;
+import org.eclipse.cdt.core.index.IndexFilter;
+import org.eclipse.cdt.core.model.ICProject;
+import org.eclipse.cdt.core.testplugin.CProjectHelper;
+import org.eclipse.cdt.core.testplugin.util.BaseTestCase;
+import org.eclipse.cdt.core.testplugin.util.TestSourceReader;
+import org.eclipse.cdt.internal.core.CCoreInternals;
+import org.eclipse.cdt.internal.core.pdom.PDOM;
+import org.eclipse.cdt.internal.core.pdom.dom.IPDOMIterator;
+import org.eclipse.cdt.internal.core.pdom.dom.PDOMBinding;
+import org.eclipse.cdt.internal.core.pdom.dom.PDOMName;
+import org.eclipse.cdt.internal.core.pdom.indexer.IndexerPreferences;
+import org.eclipse.core.resources.IProject;
+import org.eclipse.core.resources.IResource;
+import org.eclipse.core.runtime.NullProgressMonitor;
+
+public class PDOMNameTests extends BaseTestCase {
+	private ICProject cproject;
+
+	public static Test suite() {
+		return suite(PDOMNameTests.class);
+	}
+
+	@Override
+	protected void setUp() throws Exception {
+		super.setUp();
+		cproject= CProjectHelper.createCCProject("PDOMNameTest"+System.currentTimeMillis(), "bin", IPDOMManager.ID_FAST_INDEXER);
+		waitForIndexer(cproject);
+	}
+
+	@Override
+	protected void tearDown() throws Exception {
+		if (cproject != null) {
+			cproject.getProject().delete(IResource.FORCE | IResource.ALWAYS_DELETE_PROJECT_CONTENT, new NullProgressMonitor());
+		}
+		super.tearDown();
+	}
+
+	public void testExternalReferences() throws Exception {
+		IProject project = cproject.getProject();
+		TestSourceReader.createFile(project, "file.cpp", "extern void func_cpp() { func_cpp(); }");
+		TestSourceReader.createFile(project, "file.c", "extern void func_c() { func_c(); }");
+
+		IndexerPreferences.set(project, IndexerPreferences.KEY_INDEXER_ID, IPDOMManager.ID_FAST_INDEXER);
+		CCorePlugin.getIndexManager().reindex(cproject);
+		waitForIndexer(cproject);
+
+		PDOM pdom = (PDOM) CCoreInternals.getPDOMManager().getPDOM(cproject);
+		pdom.acquireWriteLock();
+		try {
+			IIndexBinding[] bindings = pdom.findBindings(new char[][]{"func_cpp".toCharArray()}, IndexFilter.ALL, npm());
+			assertEquals(1, bindings.length);
+			assertTrue(bindings[0] instanceof PDOMBinding);
+
+			PDOMBinding binding_cpp = (PDOMBinding) bindings[0];
+			PDOMName name_cpp = binding_cpp.getFirstReference();
+			assertNotNull(name_cpp);
+			assertSame(binding_cpp.getLinkage(), name_cpp.getLinkage());
+
+			bindings = pdom.findBindings(new char[][]{"func_c".toCharArray()}, IndexFilter.ALL, npm());
+			assertEquals(1, bindings.length);
+			assertTrue(bindings[0] instanceof PDOMBinding);
+
+			PDOMBinding binding_c = (PDOMBinding) bindings[0];
+			PDOMName name_c = binding_c.getFirstReference();
+			assertNotNull(name_c);
+			assertSame(binding_c.getLinkage(), name_c.getLinkage());
+
+			// Check that the external references list is currently empty.
+			IPDOMIterator<PDOMName> extNames = binding_cpp.getExternalReferences();
+			assertNotNull(extNames);
+			assertFalse(extNames.hasNext());
+
+			// Make sure the C++ binding and the C name are in different linkages, then add the name
+			// as an external reference of the binding.
+			assertNotSame(binding_cpp.getLinkage(), name_c.getLinkage());
+			binding_cpp.addReference(name_c);
+
+			// Make sure there is an external reference, then retrieve it.  Then make sure there
+			// aren't anymore external references.
+			extNames = binding_cpp.getExternalReferences();
+			assertNotNull(extNames);
+			assertTrue(extNames.hasNext());
+			PDOMName extRef = extNames.next();
+			assertNotNull(extRef);
+			assertFalse(extNames.hasNext());
+
+			// Check that the external reference is the same as the C name that was added, that the
+			// external reference does not have the same linkage as the binding, and that it does
+			// have the same linkage as the initial name.
+			assertSame(name_c.getLinkage(), extRef.getLinkage());
+			assertEquals(name_c.getRecord(), extRef.getRecord());
+			assertNotSame(binding_cpp.getLinkage(), extRef.getLinkage());
+			assertSame(binding_c.getLinkage(), extRef.getLinkage());
+		} finally {
+			pdom.releaseWriteLock();
+		}
+	}
+}

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/pdom/tests/PDOMTests.java

@@ -30,6 +30,7 @@ public class PDOMTests extends TestSuite {
 		suite.addTest(PDOMCPPBugsTest.suite());
 		suite.addTest(PDOMSearchTest.suite());
 		suite.addTest(PDOMLocationTests.suite());
+		suite.addTest(PDOMNameTests.suite());
 		suite.addTest(PDOMProviderTests.suite());
 		suite.addTest(EnumerationTests.suite());
 		suite.addTest(ClassTests.suite());

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/PDOM.java

@@ -76,6 +76,7 @@ import org.eclipse.cdt.internal.core.pdom.db.IBTreeComparator;
 import org.eclipse.cdt.internal.core.pdom.db.IBTreeVisitor;
 import org.eclipse.cdt.internal.core.pdom.dom.BindingCollector;
 import org.eclipse.cdt.internal.core.pdom.dom.FindBinding;
+import org.eclipse.cdt.internal.core.pdom.dom.IPDOMIterator;
 import org.eclipse.cdt.internal.core.pdom.dom.IPDOMLinkageFactory;
 import org.eclipse.cdt.internal.core.pdom.dom.MacroContainerCollector;
 import org.eclipse.cdt.internal.core.pdom.dom.MacroContainerPatternCollector;
@@ -242,10 +243,11 @@ public class PDOM extends PlatformObject implements IPDOM {
 	 *
 	 *  CDT 8.3 development (versions not supported on the 8.2.x branch)
 	 *  160.0 - Store specialized template parameters of class/function template specializations, bug 407497.
+	 *  161.0 - Allow reference to PDOMBinding from other PDOMLinkages, bug xyz.
 	 */
-	private static final int MIN_SUPPORTED_VERSION= version(160, 0);
-	private static final int MAX_SUPPORTED_VERSION= version(160, Short.MAX_VALUE);
-	private static final int DEFAULT_VERSION = version(160, 0);
+	private static final int MIN_SUPPORTED_VERSION= version(161, 0);
+	private static final int MAX_SUPPORTED_VERSION= version(161, Short.MAX_VALUE);
+	private static final int DEFAULT_VERSION = version(161, 0);
 
 	private static int version(int major, int minor) {
 		return (major << 16) + minor;
@@ -1166,6 +1168,12 @@ public class PDOM extends PlatformObject implements IPDOM {
 					names.add(name);
 				}
 			}
+			IPDOMIterator<PDOMName> iterator = pdomBinding.getExternalReferences();
+			while(iterator.hasNext()) {
+				name = iterator.next();
+				if (isCommitted(name))
+					names.add(name);
+			}
 		}
 	}
 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/PDOMExternalReferencesList.java

@@ -0,0 +1,184 @@
+/*
+ * Copyright (c) 2013 QNX Software Systems and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.eclipse.cdt.internal.core.pdom.db;
+
+import java.util.NoSuchElementException;
+
+import org.eclipse.cdt.core.CCorePlugin;
+import org.eclipse.cdt.internal.core.pdom.PDOM;
+import org.eclipse.cdt.internal.core.pdom.dom.IPDOMIterator;
+import org.eclipse.cdt.internal.core.pdom.dom.PDOMLinkage;
+import org.eclipse.cdt.internal.core.pdom.dom.PDOMName;
+import org.eclipse.core.runtime.CoreException;
+
+/**
+ * A utility class for storing a list of external references.  An external reference is
+ * a PDOMName that references a PDOMBinding in a different linkage.
+ * <p>
+ * External references are stored in a singly linked list with one node for each linkage.
+ * Each node stores a list of PDOMNames.  The PDOMName.BindingList is used to store the
+ * list.
+ * <p>
+ * Each node has three fields:
+ * {
+ *     INT_SIZE linkageId;
+ *     PTR_SIZE nextNode;
+ *     PTR_SIZE nameListHead;
+ * }
+ * <p>
+ * An iterator is provided to examine all names in the list.  An iterator is needed (instead
+ * of a simple list) so that we have a chance to move to the next linkage's node when we get
+ * to the end of a name list.
+ */
+public class PDOMExternalReferencesList {
+
+	private final PDOM pdom;
+	private final long record;
+
+	/**
+	 * Create a new instance at the given location in the given PDOM.
+	 */
+	public PDOMExternalReferencesList(PDOM pdom, long record) throws CoreException {
+		this.pdom = pdom;
+		this.record = record;
+	}
+
+	/**
+	 * Return an iterator for examining all names in the external references list.  Does
+	 * not return null.
+	 */
+	public IPDOMIterator<PDOMName> getIterator() throws CoreException {
+		return new Iterator(record);
+	}
+
+	/**
+	 * Add the given name to this list.
+	 */
+	public void add(PDOMName name) throws CoreException {
+		// External references are stored in a linked list of linkages.  Each node in that list
+		// is a list that uses the PDOMName binding list.
+
+		PDOMLinkage nameLinkage = name.getLinkage();
+		int nameLinkageID = nameLinkage.getLinkageID();
+
+		// Search through the nodes to find the one for the new name's linkage.  Keep track of
+		// the record that held the last examined node so that a new node can be appended if
+		// needed.
+		long lastAddr = record;
+		long nodeRec = 0;
+		while((nodeRec = pdom.getDB().getRecPtr(lastAddr)) != 0) {
+			// Each node looks like { int linkageID; recPtr nextNode; recPtr headOfList; }
+			int linkageID = pdom.getDB().getInt(nodeRec);
+			if (linkageID == nameLinkageID)
+				break;
+
+			lastAddr = nodeRec + Database.INT_SIZE;
+		}
+
+		// If there isn't already a node for this linkage, then create a new one.
+		if (nodeRec == 0) {
+			nodeRec = pdom.getDB().malloc(Database.INT_SIZE + Database.PTR_SIZE + Database.PTR_SIZE);
+
+			// Setup the new node for this linkage and initialize the list ptr with an empty list.
+			pdom.getDB().putInt(nodeRec, nameLinkageID);
+			pdom.getDB().putRecPtr(nodeRec + Database.INT_SIZE, 0);
+			pdom.getDB().putRecPtr(nodeRec + Database.INT_SIZE + Database.PTR_SIZE, 0);
+
+			// Finally insert the new node right after the last one that was examined.
+			pdom.getDB().putRecPtr(lastAddr, nodeRec);
+		}
+
+		// If the list is not empty then modify the first element to be right after the name that
+		// is being inserted.
+		long namerec = pdom.getDB().getRecPtr(nodeRec + Database.INT_SIZE + Database.PTR_SIZE);
+		if (namerec != 0) {
+			PDOMName first = new PDOMName(nameLinkage, namerec);
+			first.setPrevInBinding(name);
+			name.setNextInBinding(first);
+		}
+
+		// Finally, make the new name the first element in the list.
+		pdom.getDB().putRecPtr(nodeRec + Database.INT_SIZE + Database.PTR_SIZE, name.getRecord());
+	}
+
+	private class Iterator implements IPDOMIterator<PDOMName> {
+		private long nodeAddr;
+		private long node;
+
+		private PDOMName next;
+
+		public Iterator(long record) throws CoreException {
+			this.nodeAddr = record;
+			this.node = 0;
+
+			// Initialize next by advancing to the first name.
+			this.next = advance();
+		}
+
+		@Override
+		public boolean hasNext() throws CoreException {
+			return next != null;
+		}
+
+		@Override
+		public PDOMName next() throws CoreException {
+			if (next == null)
+				throw new NoSuchElementException();
+
+			PDOMName ret = next;
+			next = ret.getNextInBinding();
+			if (next == null)
+				next = advance();
+			return ret;
+		}
+
+		/**
+		 * Advance to the next linkage node that has a non-empty list of names.  Return the
+		 * PDOMName at the head of the list.  This is the next name that should be returned
+		 * from #next().  Return null if there are no more linkage nodes.
+		 */
+		private PDOMName advance() throws CoreException {
+			// Look through all linkage nodes to find the next one that has a non-empty
+			// names list.
+			while(true) {
+				// Skip over all nodes that don't have any names in their list.
+				long nextNameRec = 0;
+				while(nodeAddr != 0) {
+					node = pdom.getDB().getRecPtr(nodeAddr);
+					if (node == 0)
+						return null;
+
+					nextNameRec = pdom.getDB().getRecPtr(node + Database.INT_SIZE + Database.PTR_SIZE);
+					if (nextNameRec != 0)
+						break;
+					nodeAddr = node + Database.INT_SIZE;
+				}
+
+				// If nothing is found then there is no more iterating.
+				if (nodeAddr == 0
+				 || nextNameRec == 0)
+					return null;
+
+				// If a node is found that has a name in the list, then update this iterator to
+				// point to the next linkage's node (this is so the next call to advance starts
+				// at the right place).
+				nodeAddr = pdom.getDB().getRecPtr(node + Database.INT_SIZE);
+
+				// Load node's linkage and use it to return the first name in this node's list.
+				int linkageID = pdom.getDB().getInt(node);
+				PDOMLinkage linkage = pdom.getLinkage(linkageID);
+				if (linkage != null)
+					return new PDOMName(linkage, nextNameRec);
+
+				// Generate a log message about the linkageID that is no longer valid, but continue
+				// to the next node.
+				CCorePlugin.log("Could not load linkage for external reference from linkageID " + linkageID); //$NON-NLS-1$
+			}
+		}
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/IPDOMIterator.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2013 QNX Software Systems and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.eclipse.cdt.internal.core.pdom.dom;
+
+import java.util.NoSuchElementException;
+
+import org.eclipse.core.runtime.CoreException;
+
+/**
+ * A generic interface for iterating through lists that are stored in the PDOM.  The
+ * difference between this interface and the standard one in java.util is that this
+ * one can throw a CoreException from either method.  Also, this one does not provide
+ * a way to remove elements.
+ */
+public interface IPDOMIterator<T> {
+
+	/**
+	 * Return true if the next call to #next will yield a value and false otherwise.
+	 *
+	 * @see java.util.Iterator#hasNext()
+	 */
+	public boolean hasNext() throws CoreException;
+
+	/**
+	 * Return the next element in the iteration.  Throws {@link NoSuchElementException} if
+	 * there are no elements left in the iteration.
+	 *
+	 * @see java.util.Iterator#next
+	 */
+	public T next() throws CoreException;
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMBinding.java

@@ -37,6 +37,7 @@ import org.eclipse.cdt.internal.core.index.IIndexScope;
 import org.eclipse.cdt.internal.core.pdom.PDOM;
 import org.eclipse.cdt.internal.core.pdom.db.Database;
 import org.eclipse.cdt.internal.core.pdom.db.IString;
+import org.eclipse.cdt.internal.core.pdom.db.PDOMExternalReferencesList;
 import org.eclipse.cdt.internal.core.pdom.tag.PDOMTaggable;
 import org.eclipse.core.runtime.CoreException;
 
@@ -50,9 +51,10 @@ public abstract class PDOMBinding extends PDOMNamedNode implements IPDOMBinding
 	private static final int FIRST_DEF_OFFSET    = PDOMNamedNode.RECORD_SIZE + 4; // size 4
 	private static final int FIRST_REF_OFFSET    = PDOMNamedNode.RECORD_SIZE + 8; // size 4
 	private static final int LOCAL_TO_FILE		 = PDOMNamedNode.RECORD_SIZE + 12; // size 4
+	private static final int FIRST_EXTREF_OFFSET = PDOMNamedNode.RECORD_SIZE + 16; // size 4
 
 	@SuppressWarnings("hiding")
-	protected static final int RECORD_SIZE = PDOMNamedNode.RECORD_SIZE + 16;
+	protected static final int RECORD_SIZE = PDOMNamedNode.RECORD_SIZE + 20;
 	private byte hasDeclaration= -1;
 
 	protected PDOMBinding(PDOMLinkage linkage, PDOMNode parent, char[] name) throws CoreException {
@@ -95,7 +97,8 @@ public abstract class PDOMBinding extends PDOMNamedNode implements IPDOMBinding
 		Database db = pdom.getDB();
 		return db.getRecPtr(record + FIRST_DECL_OFFSET) == 0
 				&& db.getRecPtr(record + FIRST_DEF_OFFSET) == 0
-				&& db.getRecPtr(record + FIRST_REF_OFFSET) == 0;
+				&& db.getRecPtr(record + FIRST_REF_OFFSET) == 0
+				&& db.getRecPtr(record + FIRST_EXTREF_OFFSET) == 0;
 	}
 
 	@Override
@@ -132,6 +135,14 @@ public abstract class PDOMBinding extends PDOMNamedNode implements IPDOMBinding
 	}
 
 	public final void addReference(PDOMName name) throws CoreException {
+		// This needs to filter between the local and external lists because it can be used in
+		// contexts that don't know which type of list they are iterating over.  E.g., this is
+		// used when deleting names from a PDOMFile.
+		if (!getLinkage().equals(name.getLinkage())) {
+			new PDOMExternalReferencesList(getPDOM(), record + FIRST_EXTREF_OFFSET).add(name);
+			return;
+		}
+
 		PDOMName first = getFirstReference();
 		if (first != null) {
 			first.setPrevInBinding(name);
@@ -165,7 +176,25 @@ public abstract class PDOMBinding extends PDOMNamedNode implements IPDOMBinding
 		return namerec != 0 ? new PDOMName(getLinkage(), namerec) : null;
 	}
 
+	/**
+	 * Returns an iterator over the names in other linkages that reference this binding.  Does
+	 * not return null.
+	 */
+	public IPDOMIterator<PDOMName> getExternalReferences() throws CoreException {
+		return new PDOMExternalReferencesList(getPDOM(), record + FIRST_EXTREF_OFFSET).getIterator();
+	}
+
 	public void setFirstReference(PDOMName name) throws CoreException {
+		// This needs to filter between the local and external lists because it can be used in
+		// contexts that don't know which type of list they are iterating over.  E.g., this is
+		// used when deleting names from a PDOMFile.
+		if (name != null
+		 && !getLinkage().equals(name.getLinkage())) {
+			new PDOMExternalReferencesList(getPDOM(), record + FIRST_EXTREF_OFFSET).add(name);
+			return;
+		}
+
+		// Otherwise put the reference into list of locals.
 		long namerec = name != null ? name.getRecord() : 0;
 		getDB().putRecPtr(record + FIRST_REF_OFFSET, namerec);
 	}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMName.java

@@ -58,7 +58,6 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
 	public static final int READ_ACCESS 						= 0x20;
 	public static final int WRITE_ACCESS 						= 0x40;
 
-	
 	public PDOMName(PDOMLinkage linkage, IASTName name, PDOMFile file, PDOMBinding binding, PDOMName caller)
 			throws CoreException {
 		this.linkage = linkage;
@@ -112,6 +111,10 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
 		this.record = nameRecord;
 	}
 
+	public PDOMLinkage getLinkage() {
+		return linkage;
+	}
+
 	public long getRecord() {
 		return record;
 	}

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/indexview/CountNodeAction.java

@@ -26,6 +26,7 @@ import org.eclipse.cdt.internal.core.CCoreInternals;
 import org.eclipse.cdt.internal.core.pdom.IPDOM;
 import org.eclipse.cdt.internal.core.pdom.PDOM;
 import org.eclipse.cdt.internal.core.pdom.db.IBTreeVisitor;
+import org.eclipse.cdt.internal.core.pdom.dom.IPDOMIterator;
 import org.eclipse.cdt.internal.core.pdom.dom.PDOMBinding;
 import org.eclipse.cdt.internal.core.pdom.dom.PDOMFile;
 import org.eclipse.cdt.internal.core.pdom.dom.PDOMMacro;
@@ -115,6 +116,10 @@ public class CountNodeAction extends IndexAction {
 										.getFirstReference(); name != null; name = name
 										.getNextInBinding())
 									++count[REFS];
+								for (IPDOMIterator<PDOMName> i = binding.getExternalReferences();
+									 i.hasNext();
+									 i.next())
+									++count[REFS];
 								for (PDOMName name = binding
 										.getFirstDeclaration(); name != null; name = name
 										.getNextInBinding())