See 180164, reduce amount of disk-write operations when indexing.

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/pdom/tests/DBTest.java

@@ -249,7 +249,7 @@ public class DBTest extends BaseTestCase {
 		IString biss = db.newString(b);
 		IString aisc = db.newString(acs);
 		IString bisc = db.newString(bcs);
-		db.setReadOnly();
+		db.setReadOnly(true);
 		
 		assertSignEquals(expected, aiss.compare(bcs, caseSensitive));
 		assertSignEquals(expected, aiss.compare(biss, caseSensitive));

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/pdom/tests/PDOMBugsTest.java

@@ -66,7 +66,7 @@ public class PDOMBugsTest extends BaseTestCase {
 			wpdom.setProperty("c", "e");
 			assertEquals("e", wpdom.getProperty("c"));
 		} finally {
-			pdom.releaseWriteLock(0);
+			pdom.releaseWriteLock(0, true);
 		}
 	}
 	

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/index/IWritableIndex.java

@@ -67,8 +67,16 @@ public interface IWritableIndex extends IIndex {
 
 	/**
 	 * Releases a write lock, reestablishing a certain amount of read locks.
+	 * Fully equivalent to <code>releaseWriteLock(int, true)</code>.
 	 */
 	void releaseWriteLock(int establishReadLockCount);
+
+	/**
+	 * Releases a write lock, reestablishing a certain amount of read locks.
+	 * @param establishReadLockCount amount of read-locks to establish.
+	 * @param flushDatabase when true the changes are flushed to disk.
+	 */
+	void releaseWriteLock(int establishReadLockCount, boolean flushDatabase);
 	
 	/**
 	 * Resets the counters for cache-hits
@@ -90,4 +98,9 @@ public interface IWritableIndex extends IIndex {
 	 * no writable fragment.
 	 */
 	IWritableIndexFragment getPrimaryWritableFragment();
+
+	/**
+	 * Flushes all caches to the disk. 
+	 */
+	void flush() throws CoreException;
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/index/IWritableIndexFragment.java

@@ -60,8 +60,10 @@ public interface IWritableIndexFragment extends IIndexFragment {
 	
 	/**
 	 * Releases a write lock, reestablishing a certain amount of read locks.
+	 * @param establishReadLockCount amount of read-locks to establish
+	 * @param flush if <code>true</code> changes are flushed to disk
 	 */
-	void releaseWriteLock(int establishReadLockCount);
+	void releaseWriteLock(int establishReadLockCount, boolean flush);
 	
 	/**
 	 * Write the key, value mapping to the fragment properties. If a mapping for the
@@ -72,4 +74,9 @@ public interface IWritableIndexFragment extends IIndexFragment {
 	 * @throws NullPointerException if key is null
 	 */
 	public void setProperty(String propertyName, String value) throws CoreException;
+
+	/**
+	 * Flushes caches to disk.
+	 */
+	void flush() throws CoreException;
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/index/WritableCIndex.java

@@ -113,24 +113,37 @@ public class WritableCIndex extends CIndex implements IWritableIndex {
 				// rollback
 				fIsWriteLocked= false;
 				while (--i >= 0) {
-					fWritableFragments[i].releaseWriteLock(giveupReadlockCount);
+					fWritableFragments[i].releaseWriteLock(giveupReadlockCount, false);
 				}
 			}
 		}
 	}
 
 	public synchronized void releaseWriteLock(int establishReadlockCount) {
+		releaseWriteLock(establishReadlockCount, true);
+	}
+
+	public synchronized void releaseWriteLock(int establishReadlockCount, boolean flush) {
 		assert fIsWriteLocked: "No write lock to be released"; //$NON-NLS-1$
 		assert establishReadlockCount == getReadLockCount(): "Unexpected read lock is not allowed"; //$NON-NLS-1$
 
 		fIsWriteLocked= false;
-		int i= 0;
-		for (i = 0; i < fWritableFragments.length; i++) {
-			fWritableFragments[i].releaseWriteLock(establishReadlockCount);
+		for (int i = 0; i < fWritableFragments.length; i++) {
+			fWritableFragments[i].releaseWriteLock(establishReadlockCount, flush);
 		}
 	}
 	
 	public IWritableIndexFragment getPrimaryWritableFragment() {
 		return fWritableFragments.length > 0 ? fWritableFragments[0] : null;
 	}
+
+	public void flush() throws CoreException {
+		assert !fIsWriteLocked;
+		int i= 0;
+		for (i = 0; i < fWritableFragments.length; i++) {
+			fWritableFragments[i].flush();
+		}
+	}
+	
+	
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/PDOM.java

@@ -564,12 +564,12 @@ public class PDOM extends PlatformObject implements IIndexFragment, IPDOM {
 	}
 
 	final public void releaseWriteLock() {
-		releaseWriteLock(0);
+		releaseWriteLock(0, true);
 	}
-
-	public void releaseWriteLock(int establishReadLocks) {
+	
+	public void releaseWriteLock(int establishReadLocks, boolean flush) {
 		try {
-			db.setReadOnly();
+			db.setReadOnly(flush);
 		} catch (CoreException e) {
 			CCorePlugin.log(e);
 		}
@@ -736,5 +736,9 @@ public class PDOM extends PlatformObject implements IIndexFragment, IPDOM {
 
 	public void resetCacheCounters() {
 		db.resetCacheCounters();
+	}
+
+	public void flush() throws CoreException {
+		db.flush();
 	}		
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/PDOMManager.java

@@ -1037,8 +1037,8 @@ public class PDOMManager implements IWritableIndexManager, IListener {
 			PDOM pdom= (PDOM) getPDOM(cproject);
 			pdom.acquireWriteLock();
 			try {
+				pdom.flush();
 				Database db= pdom.getDB();
-				db.flushDirtyChunks();
 				FileChannel from= db.getChannel();
 				FileChannel to = new FileOutputStream(targetLocation).getChannel();
 				from.transferTo(0, from.size(), to);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/PDOMWriter.java

@@ -108,6 +108,17 @@ abstract public class PDOMWriter {
 	 */
 	protected abstract IIndexFileLocation findLocation(String absolutePath);
 		
+	/**
+	 * Fully equivalent to 
+	 * <code>addSymbols(IASTTranslationUnit, IWritableIndex, int, true, int, IProgressMonitor)</code>.
+	 * @since 4.0
+	 */
+	public void addSymbols(IASTTranslationUnit ast, 
+			IWritableIndex index, int readlockCount,
+			int configHash, IProgressMonitor pm) throws InterruptedException, CoreException {
+		addSymbols(ast, index, readlockCount, true, configHash, pm);
+	}
+
 	/**
 	 * Extracts symbols from the given ast and adds them to the index. It will
 	 * make calls to 	  
@@ -115,10 +126,14 @@ abstract public class PDOMWriter {
 	 * {@link #postAddToIndex(IIndexFileLocation, IIndexFile)},
 	 * {@link #getLastModified(IIndexFileLocation)} and
 	 * {@link #findLocation(String)} to obtain further information.
+	 * 
+	 * When flushIndex is set to <code>false</code>, you must make sure to flush the 
+	 * index after your last write operation.
 	 * @since 4.0
 	 */
-	public void addSymbols(IASTTranslationUnit ast, IWritableIndex index, int readlockCount, int configHash,
-			IProgressMonitor pm) throws InterruptedException, CoreException {
+	public void addSymbols(IASTTranslationUnit ast, 
+			IWritableIndex index, int readlockCount, boolean flushIndex,
+			int configHash, IProgressMonitor pm) throws InterruptedException, CoreException {
 		final Map symbolMap= new HashMap();
 		try {
 			HashSet contextIncludes= new HashSet();
@@ -187,7 +202,7 @@ abstract public class PDOMWriter {
 					}
 				}
 			} finally {
-				index.releaseWriteLock(readlockCount);
+				index.releaseWriteLock(readlockCount, flushIndex);
 			}
 			fStatistics.fAddToIndexTime+= System.currentTimeMillis()-start;
 		}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/TeamPDOMImportOperation.java

@@ -282,7 +282,7 @@ public class TeamPDOMImportOperation implements IWorkspaceRunnable {
 			}
 		}
 		finally {
-			pdom.releaseWriteLock(giveupReadlocks);
+			pdom.releaseWriteLock(giveupReadlocks, true);
 		}
 	}
 	

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/Chunk.java

@@ -28,20 +28,24 @@ final class Chunk {
 	
 	boolean fCacheHitFlag= false;
 	boolean fDirty= false;
-	boolean fWritable= false;
+	boolean fLocked= false;	// locked chunks must not be released from cache.
 	int fCacheIndex= -1;
 		
-	Chunk(Database db, int sequenceNumber) throws CoreException {
+	Chunk(Database db, int sequenceNumber) {
 		fDatabase= db;
 		fBuffer= ByteBuffer.allocate(Database.CHUNK_SIZE);
 		fSequenceNumber= sequenceNumber;
+	}
+
+	void read() throws CoreException {
 		try {
+			fBuffer.position(0);
 			fDatabase.getChannel().read(fBuffer, fSequenceNumber*Database.CHUNK_SIZE);
 		} catch (IOException e) {
 			throw new CoreException(new DBStatus(e));
 		}
 	}
-	
+
 	void flush() throws CoreException {
 		try {
 			fBuffer.position(0);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/ChunkCache.java

@@ -30,9 +30,9 @@ public final class ChunkCache {
 		fPageTable= new Chunk[computeLength(maxSize)];
 	}
 	
-	public synchronized void add(Chunk chunk, boolean writable) {
-		if (writable) {
-			chunk.fWritable= true;
+	public synchronized void add(Chunk chunk, boolean locked) {
+		if (locked) {
+			chunk.fLocked= true;
 		}
 		if (chunk.fCacheIndex >= 0) {
 			chunk.fCacheHitFlag= true;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/Database.java

@@ -95,7 +95,7 @@ public class Database {
 				setWritable();
 				createNewChunk();
 				setVersion(version);
-				setReadOnly();
+				setReadOnly(true);
 			}
 		} catch (IOException e) {
 			throw new CoreException(new DBStatus(e));
@@ -158,24 +158,25 @@ public class Database {
 		
 		// for performance reasons try to find chunk and mark it without
 		// synchronizing. This means that we might pick up a chunk that
-		// has been paged out, which is ok.
-		// Furthermore the hitflag may not be seen by the clock-alorithm,
+		// has been paged out, which is fine.
+		// Furthermore the hit-flag may not be seen by the clock-algorithm,
 		// which might lead to the eviction of a chunk. With the next
 		// cache failure we are in sync again, though.
 		Chunk chunk = chunks[index];		
-		if (chunk != null && chunk.fWritable == fWritable) {
+		if (chunk != null && (chunk.fLocked || !fWritable)) {
 			chunk.fCacheHitFlag= true;
 			cacheHits++;
 			return chunk;
 		}
 		
 		// here is the safe code that has to be performed if we cannot
-		// get ahold of the chunk.
+		// get hold of the chunk.
 		synchronized(fCache) {
 			chunk= chunks[index];
 			if (chunk == null) {
 				cacheMisses++;
 				chunk = chunks[index] = new Chunk(this, index);
+				chunk.read();
 			}
 			else {
 				cacheHits++;
@@ -239,18 +240,16 @@ public class Database {
 	}
 	
 	private int createNewChunk() throws CoreException {
-		try {
-			Chunk[] oldtoc = chunks;
-			int n = oldtoc.length;
-			int offset = n * CHUNK_SIZE;
-			file.seek(offset);
-			file.write(new byte[CHUNK_SIZE]);
-			chunks = new Chunk[n + 1];
-			System.arraycopy(oldtoc, 0, chunks, 0, n);
-			return offset;
-		} catch (IOException e) {
-			throw new CoreException(new DBStatus(e));
-		}
+		Chunk[] oldtoc = chunks;
+		int n = oldtoc.length;
+		int offset = n * CHUNK_SIZE;
+		chunks = new Chunk[n + 1];
+		System.arraycopy(oldtoc, 0, chunks, 0, n);
+		final Chunk chunk= new Chunk(this, n);
+		chunk.fDirty= true;
+		chunks[n]= chunk;
+		fCache.add(chunk, true);
+		return offset;
 	}
 	
 	private int getFirstBlock(int blocksize) throws CoreException {
@@ -390,12 +389,12 @@ public class Database {
 	/**
 	 * Closes the database. 
 	 * <p>
-	 * The behaviour of any further calls to the Database is undefined
+	 * The behavior of any further calls to the Database is undefined
 	 * @throws IOException
 	 * @throws CoreException 
 	 */
 	public void close() throws CoreException {
-		setReadOnly();
+		setReadOnly(true);
 		removeChunksFromCache();
 		chunks= new Chunk[0];
 		try {
@@ -415,9 +414,10 @@ public class Database {
 	/**
 	 * Called from any thread via the cache, protected by {@link #fCache}.
 	 */
-	void releaseChunk(Chunk chunk) {
-		if (!chunk.fWritable)
+	void releaseChunk(final Chunk chunk) {
+		if (!chunk.fLocked) {
 			chunks[chunk.fSequenceNumber]= null;
+		}			
 	}
 
 	/**
@@ -432,38 +432,88 @@ public class Database {
 		fWritable= true;
 	}
 
-	public void setReadOnly() throws CoreException {
+	public void setReadOnly(final boolean flush) throws CoreException {
 		if (fWritable) {
 			fWritable= false;
-			flushDirtyChunks();
-		}
-	}
-	
-	public void flushDirtyChunks() throws CoreException {
-		ArrayList dirtyChunks= new ArrayList();
-		synchronized (fCache) {
-			for (int i = 0; i < chunks.length; i++) {
-				Chunk chunk= chunks[i];
-				if (chunk != null && chunk.fWritable) {
-					chunk.fWritable= false;
-					if (chunk.fCacheIndex < 0) {
-						chunks[i]= null;
-					}
-					if (chunk.fDirty) {
-						dirtyChunks.add(chunk);
+			
+			ArrayList dirtyChunks= new ArrayList();
+			synchronized (fCache) {
+				for (int i= chunks.length-1; i >= 0 ; i--) {
+					Chunk chunk= chunks[i];
+					if (chunk != null) {
+						if (chunk.fCacheIndex < 0) {
+							chunk.fLocked= false;
+							chunks[i]= null;
+							if (chunk.fDirty) {
+								dirtyChunks.add(chunk);
+							}
+						}
+						else if (chunk.fLocked) {
+							if (!chunk.fDirty) {
+								chunk.fLocked= false;
+							}
+							else if (flush) {
+								chunk.fLocked= false;
+								dirtyChunks.add(chunk);
+							}
+						}
+						else if (flush && chunk.fDirty) {
+							dirtyChunks.add(chunk);
+						}
 					}
 				}
 			}
+			
+			if (!dirtyChunks.isEmpty()) {
+				for (Iterator it = dirtyChunks.iterator(); it.hasNext();) {
+					Chunk chunk = (Chunk) it.next();
+					chunk.flush();
+				}
+			}
 		}
-		
+	}
+	
+	public void flush() throws CoreException {
+		if (fWritable) {
+			try {
+				setReadOnly(true);
+			}
+			finally {
+				setWritable();
+			}
+			return;
+		}
+
+		// be careful as other readers may access chunks concurrently
+		ArrayList dirtyChunks= new ArrayList();
+		synchronized (fCache) {
+			for (int i= chunks.length-1; i >= 0 ; i--) {
+				Chunk chunk= chunks[i];
+				if (chunk != null && chunk.fDirty) {
+					dirtyChunks.add(chunk);
+				}
+			}
+		}
+
 		if (!dirtyChunks.isEmpty()) {
 			for (Iterator it = dirtyChunks.iterator(); it.hasNext();) {
 				Chunk chunk = (Chunk) it.next();
 				chunk.flush();
 			}
 		}
+		
+		// only after the chunks are flushed we may unlock and release them.
+		synchronized (fCache) {
+			for (Iterator it = dirtyChunks.iterator(); it.hasNext();) {
+				Chunk chunk = (Chunk) it.next();
+				chunk.fLocked= false;
+				if (chunk.fCacheIndex < 0) {
+					chunks[chunk.fSequenceNumber]= null;
+				}
+			}
+		}
 	}
-	
+		
 	public void resetCacheCounters() {
 		cacheHits= cacheMisses= 0;
 	}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/export/GeneratePDOM.java

@@ -69,18 +69,22 @@ public class GeneratePDOM implements ISafeRunnable {
 		try {
 			CCoreInternals.getPDOMManager().exportProjectPDOM(cproject, targetLocation, converter);
 			WritablePDOM exportedPDOM= new WritablePDOM(targetLocation, converter, LanguageManager.getInstance().getPDOMLinkageFactoryMappings());
-
-			exportedPDOM.acquireWriteLock(0);
 			try {
-				Map exportProperties= pm.getExportProperties();
-				if(exportProperties!=null) {
-					for(Iterator i = exportProperties.entrySet().iterator(); i.hasNext(); ) {
-						Map.Entry entry = (Map.Entry) i.next();
-						exportedPDOM.setProperty((String) entry.getKey(), (String) entry.getValue());
+				exportedPDOM.acquireWriteLock(0);
+				try {
+					Map exportProperties= pm.getExportProperties();
+					if(exportProperties!=null) {
+						for(Iterator i = exportProperties.entrySet().iterator(); i.hasNext(); ) {
+							Map.Entry entry = (Map.Entry) i.next();
+							exportedPDOM.setProperty((String) entry.getKey(), (String) entry.getValue());
+						}
 					}
+				} finally {
+					exportedPDOM.releaseWriteLock(0, true);
 				}
-			} finally {
-				exportedPDOM.releaseWriteLock(0);
+			}
+			finally {
+				exportedPDOM.close();
 			}
 		} catch(InterruptedException ie) {
 			String msg= MessageFormat.format(Messages.GeneratePDOM_GenericGenerationFailed, new Object[] {ie.getMessage()});

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/PDOMIndexerTask.java

@@ -176,6 +176,15 @@ public abstract class PDOMIndexerTask extends PDOMWriter implements IPDOMIndexer
 	 * @since 4.0
 	 */
 	protected void parseTUs(IWritableIndex index, int readlockCount, Collection sources, Collection headers, IProgressMonitor monitor) throws CoreException, InterruptedException {
+		try {
+			internalParseTUs(index, readlockCount, sources, headers, monitor);
+		}
+		finally {
+			index.flush();
+		}
+	}
+	
+	private void internalParseTUs(IWritableIndex index, int readlockCount, Collection sources, Collection headers, IProgressMonitor monitor) throws CoreException, InterruptedException {
 		int options= 0;
 		if (checkProperty(IndexerPreferences.KEY_SKIP_ALL_REFERENCES)) {
 			options |= AbstractLanguage.OPTION_SKIP_FUNCTION_BODIES;
@@ -273,7 +282,7 @@ public abstract class PDOMIndexerTask extends PDOMWriter implements IPDOMIndexer
 					IASTTranslationUnit ast= createAST(tu, scanner, options, pm);
 					fStatistics.fParsingTime += System.currentTimeMillis()-start;
 					if (ast != null) {
-						addSymbols(ast, index, readlockCount, configHash, pm);
+						addSymbols(ast, index, readlockCount, false, configHash, pm);
 					}
 				}
 			}
@@ -330,7 +339,7 @@ public abstract class PDOMIndexerTask extends PDOMWriter implements IPDOMIndexer
 				
 			fStatistics.fParsingTime += System.currentTimeMillis()-start;
 			if (ast != null) {
-				addSymbols(ast, index, readlockCount, 0, pm);
+				addSymbols(ast, index, readlockCount, false, 0, pm);
 				updateInfo(-1, +1, 0);
 			}
 		}