Bug 393513. Fixed serialization of very large types and template

arguments.

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/PDOM.java

@@ -218,6 +218,7 @@ public class PDOM extends PlatformObject implements IPDOM {
 	 *  124.0 - GCC attributes and NO_RETURN flag for functions.
 	 *  #125.0# - Indexes for unresolved includes and files indexed with I/O errors. <<CDT 8.1>>
 <<<<<<< cdt_8_1
+<<<<<<< cdt_8_1
 <<<<<<< cdt_8_1
 	 *  126.0 - Dependent expressions, bug 299911.
 	 *  127.0 - Explicit virtual overrides, bug 380623.

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/Chunk.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2005, 2009 QNX Software Systems and others.
+ * Copyright (c) 2005, 2012 QNX Software Systems and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -9,6 +9,7 @@
  *     QNX - Initial API and implementation
  *     Markus Schorn (Wind River Systems)
  *     IBM Corporation
+ *     Sergey Prigogin (Google)
  *******************************************************************************/
 package org.eclipse.cdt.internal.core.pdom.db;
 
@@ -301,21 +302,22 @@ final class Chunk {
 	}
 
 	void put(final long offset, final byte[] data, final int len) {
+		put(offset, data, 0, len);
+	}
+	
+	void put(final long offset, final byte[] data, int dataPos, final int len) {
 		assert fLocked;
 		fDirty = true;
 		int idx = recPtrToIndex(offset);
-		int i= 0;
+		System.arraycopy(data, dataPos, fBuffer, idx, len);
-		while (i < len) {
-			fBuffer[idx++]= data[i++];
-		}
 	}
 	
 	public void get(final long offset, byte[] data) {
+		get(offset, data, 0, data.length);
+	}
+
+	public void get(final long offset, byte[] data, int dataPos, int len) {
 		int idx = recPtrToIndex(offset);
-		final int end= idx + data.length;
+		System.arraycopy(fBuffer, idx, data, dataPos, len);
-		int i= 0;
-		while (idx < end) {
-			data[i++]= fBuffer[idx++];
-		}
 	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/Database.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2005, 2010 QNX Software Systems and others.
+ * Copyright (c) 2005, 2012 QNX Software Systems and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -10,6 +10,7 @@
  *     Symbian - Add some non-javadoc implementation notes
  *     Markus Schorn (Wind River Systems)
  *     IBM Corporation
+ *     Sergey Prigogin (Google)
  *******************************************************************************/
 package org.eclipse.cdt.internal.core.pdom.db;
 
@@ -530,10 +531,18 @@ public class Database {
 		getChunk(offset).put(offset, data, len);
 	}
 
+	public void putBytes(long offset, byte[] data, int dataPos, int len) throws CoreException {
+		getChunk(offset).put(offset, data, dataPos, len);
+	}
+
 	public void getBytes(long offset, byte[] data) throws CoreException {
 		getChunk(offset).get(offset, data);
 	}
 
+	public void getBytes(long offset, byte[] data, int dataPos, int len) throws CoreException {
+		getChunk(offset).get(offset, data, dataPos, len);
+	}
+
 	public IString newString(String string) throws CoreException {
 		return newString(string.toCharArray());
 	}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMLinkage.java

@@ -11,6 +11,7 @@
  *     IBM Corporation
  *     Andrew Ferguson (Symbian)
  *     Jens Elmenthaler - http://bugs.eclipse.org/173458 (camel case completion)
+ *     Sergey Prigogin (Google)
  *******************************************************************************/
 package org.eclipse.cdt.internal.core.pdom.dom;
 
@@ -168,7 +169,6 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 		}
 	}
 
-
 	@Override
 	public void addChild(PDOMNode child) throws CoreException {
 		getIndex().insert(child.getRecord());
@@ -438,34 +438,85 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 		if (type != null) {
 			TypeMarshalBuffer bc= new TypeMarshalBuffer(this);
 			bc.marshalType(type);
-			int len= bc.getPosition();
+			storeBuffer(db, offset, bc, Database.TYPE_SIZE);
+		}
+	}
+
+	private void storeBuffer(Database db, long offset, TypeMarshalBuffer buf, int maxInlineSize) throws CoreException {
+		int len= buf.getPosition();
 		if (len > 0) {
-				if (len <= Database.TYPE_SIZE) {
+			if (len <= maxInlineSize) {
-					db.putBytes(offset, bc.getBuffer(), len);
+				db.putBytes(offset, buf.getBuffer(), len);
-				} else if (len <= Database.MAX_MALLOC_SIZE-2){
+			} else  {
-					long ptr= db.malloc(len+2);
-					db.putShort(ptr, (short) len);
-					db.putBytes(ptr+2, bc.getBuffer(), len);
 				db.putByte(offset, TypeMarshalBuffer.INDIRECT_TYPE);
-					db.putRecPtr(offset+2, ptr);
+				long chainOffset = offset + 1;
+				int bufferPos = 0;
+				while (bufferPos < len) {
+					int chunkLength = len - bufferPos + 2;
+					boolean chainingRequired = false;
+					if (chunkLength > Database.MAX_MALLOC_SIZE) {
+						chunkLength = Database.MAX_MALLOC_SIZE;
+						chainingRequired = true;
+					}
+					long ptr = db.malloc(chunkLength);
+					db.putRecPtr(chainOffset, ptr);
+					db.putShort(ptr, (short) len);
+					int pos = 2;
+					if (chainingRequired) {
+						// Reserve space for the chaining pointer.
+						chainOffset = ptr + 2; pos += Database.PTR_SIZE;
+					}
+					chunkLength -= pos;
+					db.putBytes(ptr + pos, buf.getBuffer(), bufferPos, chunkLength);
+					bufferPos += chunkLength;
 				}
 			}
 		}
 	}
 
+	private byte[] loadLinkedSerializedData(final Database db, long offset) throws CoreException {
+		long ptr= db.getRecPtr(offset);
+		int len= db.getShort(ptr) & 0xffff;
+		byte[] data= new byte[len];
+		int bufferPos = 0;
+		while (bufferPos < len) {
+			int chunkLength = len - bufferPos + 2;
+			int pos = 2;
+			long chunkPtr = ptr;
+			if (chunkLength > Database.MAX_MALLOC_SIZE) {
+				chunkLength = Database.MAX_MALLOC_SIZE;
+				ptr= db.getRecPtr(chunkPtr + pos); pos += Database.PTR_SIZE;
+			}
+			chunkLength -= pos;
+			db.getBytes(chunkPtr + pos, data, bufferPos, chunkLength);
+			bufferPos += chunkLength;
+		}
+		return data;
+	}
+
+	private void deleteSerializedData(Database db, long offset, int maxInlineSize) throws CoreException {
+		byte firstByte= db.getByte(offset);
+		if (firstByte == TypeMarshalBuffer.INDIRECT_TYPE) {
+			long ptr= db.getRecPtr(offset + 1);
+			int len= db.getShort(ptr) & 0xffff;
+			while (len > 0) {
+				int chunkLength = len + 2;
+				int pos = 2;
+				long chunkPtr = ptr;
+				if (chunkLength > Database.MAX_MALLOC_SIZE) {
+					chunkLength = Database.MAX_MALLOC_SIZE;
+					ptr= db.getRecPtr(chunkPtr + pos); pos += Database.PTR_SIZE;
+				}
+				chunkLength -= pos;
+				db.free(chunkPtr);
+				len -= chunkLength;
+			}
+		}
+		db.clearBytes(offset, maxInlineSize);
+	}
+
 	private void deleteType(Database db, long offset) throws CoreException {
-		byte firstByte= db.getByte(offset);
+		deleteSerializedData(db, offset, Database.TYPE_SIZE);
-		if (firstByte == TypeMarshalBuffer.INDIRECT_TYPE) {
-			long ptr= db.getRecPtr(offset+2);
-			clearType(db, offset);
-			db.free(ptr);
-		} else {
-			clearType(db, offset);
-		}
-	}
-
-	private void clearType(Database db, long offset) throws CoreException {
-		db.clearBytes(offset, Database.TYPE_SIZE);
 	}
 
 	public IType loadType(long offset) throws CoreException {
@@ -474,10 +525,7 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 		byte[] data= null;
 		switch (firstByte) {
 		case TypeMarshalBuffer.INDIRECT_TYPE:
-			long ptr= db.getRecPtr(offset+2);
+			data = loadLinkedSerializedData(db, offset + 1);			
-			int len= db.getShort(ptr) & 0xffff;
-			data= new byte[len];
-			db.getBytes(ptr+2, data);
 			break;
 		case TypeMarshalBuffer.UNSTORABLE_TYPE:
 			return TypeMarshalBuffer.UNSTORABLE_TYPE_PROBLEM;
@@ -501,34 +549,12 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 		if (binding != null) {
 			TypeMarshalBuffer bc= new TypeMarshalBuffer(this);
 			bc.marshalBinding(binding);
-			int len= bc.getPosition();
+			storeBuffer(db, offset, bc, Database.TYPE_SIZE);
-			if (len > 0) {
-				if (len <= Database.TYPE_SIZE) {
-					db.putBytes(offset, bc.getBuffer(), len);
-				} else if (len <= Database.MAX_MALLOC_SIZE-2){
-					long ptr= db.malloc(len+2);
-					db.putShort(ptr, (short) len);
-					db.putBytes(ptr+2, bc.getBuffer(), len);
-					db.putByte(offset, TypeMarshalBuffer.INDIRECT_TYPE);
-					db.putRecPtr(offset+2, ptr);
-				}
-			}
 		}
 	}
 
 	private void deleteBinding(Database db, long offset) throws CoreException {
-		byte firstByte= db.getByte(offset);
+		deleteSerializedData(db, offset, Database.TYPE_SIZE);
-		if (firstByte == TypeMarshalBuffer.INDIRECT_TYPE) {
-			long ptr= db.getRecPtr(offset+2);
-			clearBinding(db, offset);
-			db.free(ptr);
-		} else {
-			clearBinding(db, offset);
-		}
-	}
-
-	private void clearBinding(Database db, long offset) throws CoreException {
-		db.clearBytes(offset, Database.TYPE_SIZE);
 	}
 
 	public IBinding loadBinding(long offset) throws CoreException {
@@ -537,10 +563,7 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 		byte[] data= null;
 		switch (firstByte) {
 		case TypeMarshalBuffer.INDIRECT_TYPE:
-			long ptr= db.getRecPtr(offset+2);
+			data = loadLinkedSerializedData(db, offset + 1);			
-			int len= db.getShort(ptr) & 0xffff;
-			data= new byte[len];
-			db.getBytes(ptr+2, data);
 			break;
 		case TypeMarshalBuffer.UNSTORABLE_TYPE:
 			return new ProblemBinding(null, ISemanticProblem.TYPE_NOT_PERSISTED);
@@ -564,34 +587,12 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 		if (arg != null) {
 			TypeMarshalBuffer bc= new TypeMarshalBuffer(this);
 			bc.marshalTemplateArgument(arg);
-			int len= bc.getPosition();
+			storeBuffer(db, offset, bc, Database.ARGUMENT_SIZE);
-			if (len > 0) {
-				if (len <= Database.ARGUMENT_SIZE) {
-					db.putBytes(offset, bc.getBuffer(), len);
-				} else if (len <= Database.MAX_MALLOC_SIZE-2){
-					long ptr= db.malloc(len+2);
-					db.putShort(ptr, (short) len);
-					db.putBytes(ptr+2, bc.getBuffer(), len);
-					db.putByte(offset, TypeMarshalBuffer.INDIRECT_TYPE);
-					db.putRecPtr(offset+2, ptr);
-				}
-			}
 		}
 	}
 
 	private void deleteArgument(Database db, long offset) throws CoreException {
-		byte firstByte= db.getByte(offset);
+		deleteSerializedData(db, offset, Database.ARGUMENT_SIZE);
-		if (firstByte == TypeMarshalBuffer.INDIRECT_TYPE) {
-			long ptr= db.getRecPtr(offset+2);
-			clearArgument(db, offset);
-			db.free(ptr);
-		} else {
-			clearArgument(db, offset);
-		}
-	}
-
-	private void clearArgument(Database db, long offset) throws CoreException {
-		db.clearBytes(offset, Database.ARGUMENT_SIZE);
 	}
 
 	public ICPPTemplateArgument loadTemplateArgument(long offset) throws CoreException {
@@ -600,16 +601,13 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 		byte[] data= null;
 		switch (firstByte) {
 		case TypeMarshalBuffer.INDIRECT_TYPE:
-			long ptr= db.getRecPtr(offset+2);
+			data = loadLinkedSerializedData(db, offset + 1);			
-			int len= db.getShort(ptr) & 0xffff;
-			data= new byte[len];
-			db.getBytes(ptr+2, data);
 			break;
 		case TypeMarshalBuffer.UNSTORABLE_TYPE:
 		case TypeMarshalBuffer.NULL_TYPE:
 			return null;
 		default:
-			data= new byte[Database.TYPE_SIZE];
+			data= new byte[Database.ARGUMENT_SIZE];
 			db.getBytes(offset, data);
 			break;
 		}
@@ -626,23 +624,12 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 		if (value != null) {
 			TypeMarshalBuffer bc= new TypeMarshalBuffer(this);
 			bc.marshalValue(value);
-			int len= bc.getPosition();
+			storeBuffer(db, offset, bc, Database.VALUE_SIZE);
-			if (len > 0) {
-				if (len <= Database.TYPE_SIZE) {
-					db.putBytes(offset, bc.getBuffer(), len);
-				} else if (len <= Database.MAX_MALLOC_SIZE-2){
-					long ptr= db.malloc(len+2);
-					db.putShort(ptr, (short) len);
-					db.putBytes(ptr+2, bc.getBuffer(), len);
-					db.putByte(offset, TypeMarshalBuffer.INDIRECT_TYPE);
-					db.putRecPtr(offset+2, ptr);
-				}
-			}
 		}
 	}
 
 	private void deleteValue(Database db, long offset) throws CoreException {
-		deleteType(db, offset);
+		deleteSerializedData(db, offset, Database.VALUE_SIZE);
 	}
 
 	public IValue loadValue(long offset) throws CoreException {
@@ -651,15 +638,12 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 		byte[] data= null;
 		switch (firstByte) {
 		case TypeMarshalBuffer.INDIRECT_TYPE:
-			long ptr= db.getRecPtr(offset+2);
+			data = loadLinkedSerializedData(db, offset + 1);			
-			int len= db.getShort(ptr) & 0xffff;
-			data= new byte[len];
-			db.getBytes(ptr+2, data);
 			break;
 		case TypeMarshalBuffer.NULL_TYPE:
 			return null;
 		default:
-			data= new byte[Database.TYPE_SIZE];
+			data= new byte[Database.VALUE_SIZE];
 			db.getBytes(offset, data);
 			break;
 		}