1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-07-01 22:25:25 +02:00

Reducing the size of the index, bug 210392.

This commit is contained in:
Markus Schorn 2007-12-13 10:49:18 +00:00
parent ad701b9da6
commit 0888279e30
7 changed files with 112 additions and 78 deletions

View file

@ -64,14 +64,16 @@ public class DBTest extends BaseTestCase {
assertEquals(0, db.getVersion());
final int realsize = 42;
final int blocksize = (realsize / Database.MIN_SIZE + 1) * Database.MIN_SIZE;
final int deltas = (realsize+Database.BLOCK_HEADER_SIZE + Database.BLOCK_SIZE_DELTA - 1) / Database.BLOCK_SIZE_DELTA;
final int blocksize = deltas * Database.BLOCK_SIZE_DELTA;
final int freeDeltas= Database.CHUNK_SIZE/Database.BLOCK_SIZE_DELTA-deltas;
int mem = db.malloc(realsize);
assertEquals(-blocksize, db.getInt(mem - Database.INT_SIZE));
assertEquals(-blocksize, db.getShort(mem - Database.BLOCK_HEADER_SIZE));
db.free(mem);
assertEquals(blocksize, db.getInt(mem - Database.INT_SIZE));
assertEquals(mem - Database.INT_SIZE, db.getInt((blocksize / Database.MIN_SIZE) * Database.INT_SIZE));
assertEquals(mem - Database.INT_SIZE + blocksize, db.getInt(((Database.CHUNK_SIZE - blocksize) / Database.MIN_SIZE) * Database.INT_SIZE));
assertEquals(blocksize, db.getShort(mem - Database.BLOCK_HEADER_SIZE));
assertEquals(mem - Database.BLOCK_HEADER_SIZE, db.getInt((deltas-Database.MIN_BLOCK_DELTAS+1) * Database.INT_SIZE));
assertEquals(mem - Database.BLOCK_HEADER_SIZE + blocksize, db.getInt((freeDeltas-Database.MIN_BLOCK_DELTAS+1) * Database.INT_SIZE));
}
public void testBug192437() throws IOException {
@ -100,16 +102,18 @@ public class DBTest extends BaseTestCase {
public void testFreeBlockLinking() throws Exception {
final int realsize = 42;
final int blocksize = (realsize / Database.MIN_SIZE + 1) * Database.MIN_SIZE;
final int deltas = (realsize+Database.BLOCK_HEADER_SIZE + Database.BLOCK_SIZE_DELTA - 1) / Database.BLOCK_SIZE_DELTA;
final int blocksize = deltas * Database.BLOCK_SIZE_DELTA;
final int freeDeltas= Database.MIN_BLOCK_DELTAS-deltas;
int mem1 = db.malloc(realsize);
int mem2 = db.malloc(realsize);
db.free(mem1);
db.free(mem2);
assertEquals(mem2 - Database.INT_SIZE, db.getInt((blocksize / Database.MIN_SIZE) * Database.INT_SIZE));
assertEquals(mem2 - Database.BLOCK_HEADER_SIZE, db.getInt((deltas-Database.MIN_BLOCK_DELTAS+1) * Database.INT_SIZE));
assertEquals(0, db.getInt(mem2));
assertEquals(mem1 - Database.INT_SIZE, db.getInt(mem2 + Database.INT_SIZE));
assertEquals(mem2 - Database.INT_SIZE, db.getInt(mem1));
assertEquals(mem1 - Database.BLOCK_HEADER_SIZE, db.getInt(mem2 + Database.INT_SIZE));
assertEquals(mem2 - Database.BLOCK_HEADER_SIZE, db.getInt(mem1));
assertEquals(0, db.getInt(mem1 + Database.INT_SIZE));
}

View file

@ -153,6 +153,7 @@ public class PDOM extends PlatformObject implements IPDOM {
* 51 - modeling extern "C" (bug 191989)
* 52 - files per linkage (bug 191989)
* 53 - polymorphic method calls (bug 156691)
* 54 - optimization of database size (bug 210392)
*/
public static final int LINKAGES = Database.DATA_AREA;

View file

@ -6,9 +6,9 @@
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* QNX - Initial API and implementation
* Markus Schorn (Wind River Systems)
* IBM Corporation
* QNX - Initial API and implementation
* Markus Schorn (Wind River Systems)
* IBM Corporation
*******************************************************************************/
package org.eclipse.cdt.internal.core.pdom.db;
@ -95,6 +95,22 @@ final class Chunk {
((fBuffer[++idx] & 0xff) << 0);
}
public void put3ByteUnsignedInt(final int offset, final int value) {
assert fLocked;
fDirty= true;
int idx= offset % Database.CHUNK_SIZE;
fBuffer[idx]= (byte)(value >> 16);
fBuffer[++idx]= (byte)(value >> 8);
fBuffer[++idx]= (byte)(value);
}
public int get3ByteUnsignedInt(final int offset) {
int idx= offset % Database.CHUNK_SIZE;
return ((fBuffer[idx] & 0xff) << 16) |
((fBuffer[++idx] & 0xff) << 8) |
((fBuffer[++idx] & 0xff) << 0);
}
public void putShort(final int offset, final short value) {
assert fLocked;
fDirty= true;

View file

@ -42,12 +42,12 @@ import org.eclipse.core.runtime.Status;
* offset content
* _____________________________
* 0 | version number
* INT_SIZE | pointer to head of linked list of blocks of size MIN_SIZE
* INT_SIZE | pointer to head of linked list of blocks of size MIN_BLOCK_DELTAS*BLOCK_SIZE_DELTA
* .. | ...
* INT_SIZE * m (1) | pointer to head of linked list of blocks of size MIN_SIZE * m
* INT_SIZE * m (1) | pointer to head of linked list of blocks of size (m+MIN_BLOCK_DELTAS) * BLOCK_SIZE_DELTA
* DATA_AREA | undefined (PDOM stores its own house-keeping data in this area)
*
* (1) where m <= (CHUNK_SIZE / MIN_SIZE)
* (1) where 2 <= m <= CHUNK_SIZE/BLOCK_SIZE_DELTA - MIN_BLOCK_DELTAS + 1
*
* ===== block structure
*
@ -59,7 +59,21 @@ import org.eclipse.core.runtime.Status;
*
*/
public class Database {
// public for tests only, you shouldn't need these
public static final int INT_SIZE = 4;
public static final int CHUNK_SIZE = 1024 * 4;
public static final int BLOCK_HEADER_SIZE= 2;
public static final int BLOCK_SIZE_DELTA= 8;
public static final int MIN_BLOCK_DELTAS = 2; // a block must at least be 2 + 2*4 bytes to link the free blocks.
public static final int MAX_BLOCK_DELTAS = CHUNK_SIZE/BLOCK_SIZE_DELTA;
public static final int MAX_MALLOC_SIZE = MAX_BLOCK_DELTAS*BLOCK_SIZE_DELTA - BLOCK_HEADER_SIZE;
public static final int VERSION_OFFSET = 0;
public static final int DATA_AREA = (CHUNK_SIZE / BLOCK_SIZE_DELTA - MIN_BLOCK_DELTAS + 2) * INT_SIZE;
private static final int BLOCK_PREV_OFFSET = BLOCK_HEADER_SIZE;
private static final int BLOCK_NEXT_OFFSET = BLOCK_HEADER_SIZE + INT_SIZE;
private final File fLocation;
private final RandomAccessFile fFile;
private boolean fExclusiveLock= false; // necessary for any write operation
@ -76,18 +90,6 @@ public class Database {
private long cacheHits;
private long cacheMisses;
// public for tests only, you shouldn't need these
public static final int VERSION_OFFSET = 0;
public static final int CHUNK_SIZE = 1024 * 4;
public static final int MIN_SIZE = 16;
public static final int INT_SIZE = 4;
public static final int CHAR_SIZE = 2;
public static final int PREV_OFFSET = INT_SIZE;
public static final int NEXT_OFFSET = INT_SIZE * 2;
public static final int DATA_AREA = CHUNK_SIZE / MIN_SIZE * INT_SIZE + INT_SIZE;
public static final int MAX_SIZE = CHUNK_SIZE - 4; // Room for overhead
/**
* Construct a new Database object, creating a backing file if necessary.
* @param location the local file path for the database
@ -210,25 +212,25 @@ public class Database {
* @param size
* @return
*/
public int malloc(int size) throws CoreException {
public int malloc(final int datasize) throws CoreException {
assert fExclusiveLock;
if (size > MAX_SIZE)
if (datasize < 0 || datasize > MAX_MALLOC_SIZE)
// Too Big
throw new CoreException(new Status(IStatus.ERROR, CCorePlugin.PLUGIN_ID, 0,
CCorePlugin.getResourceString("pdom.requestTooLarge"), new IllegalArgumentException())); //$NON-NLS-1$
int needDeltas= (datasize + BLOCK_HEADER_SIZE + BLOCK_SIZE_DELTA - 1) / BLOCK_SIZE_DELTA;
if (needDeltas < MIN_BLOCK_DELTAS) {
needDeltas= MIN_BLOCK_DELTAS;
}
// Which block size
int freeblock = 0;
int blocksize;
int matchsize = 0;
for (blocksize = MIN_SIZE; blocksize <= CHUNK_SIZE; blocksize += MIN_SIZE) {
if (blocksize - INT_SIZE >= size) {
if (matchsize == 0) // our real size
matchsize = blocksize;
freeblock = getFirstBlock(blocksize);
if (freeblock != 0)
break;
}
int useDeltas;
for (useDeltas= needDeltas; useDeltas <= MAX_BLOCK_DELTAS; useDeltas++) {
freeblock = getFirstBlock(useDeltas*BLOCK_SIZE_DELTA);
if (freeblock != 0)
break;
}
// get the block
@ -236,26 +238,29 @@ public class Database {
if (freeblock == 0) {
// allocate a new chunk
freeblock= createNewChunk();
blocksize = CHUNK_SIZE;
useDeltas = MAX_BLOCK_DELTAS;
chunk = getChunk(freeblock);
} else {
chunk = getChunk(freeblock);
removeBlock(chunk, blocksize, freeblock);
removeBlock(chunk, useDeltas*BLOCK_SIZE_DELTA, freeblock);
}
if (blocksize != matchsize) {
final int unusedDeltas = useDeltas-needDeltas;
if (unusedDeltas >= MIN_BLOCK_DELTAS) {
// Add in the unused part of our block
addBlock(chunk, blocksize - matchsize, freeblock + matchsize);
addBlock(chunk, unusedDeltas*BLOCK_SIZE_DELTA, freeblock + needDeltas*BLOCK_SIZE_DELTA);
useDeltas= needDeltas;
}
// Make our size negative to show in use
chunk.putInt(freeblock, - matchsize);
final int usedSize= useDeltas*BLOCK_SIZE_DELTA;
chunk.putShort(freeblock, (short) -usedSize);
// Clear out the block, lots of people are expecting this
chunk.clear(freeblock + 4, size);
chunk.clear(freeblock + BLOCK_HEADER_SIZE, usedSize-BLOCK_HEADER_SIZE);
malloced += matchsize;
return freeblock + 4;
malloced+= usedSize;
return freeblock + BLOCK_HEADER_SIZE;
}
private int createNewChunk() throws CoreException {
@ -276,38 +281,38 @@ public class Database {
private int getFirstBlock(int blocksize) throws CoreException {
assert fLocked;
return fHeaderChunk.getInt((blocksize / MIN_SIZE) * INT_SIZE);
return fHeaderChunk.getInt((blocksize/BLOCK_SIZE_DELTA - MIN_BLOCK_DELTAS + 1) * INT_SIZE);
}
private void setFirstBlock(int blocksize, int block) throws CoreException {
assert fExclusiveLock;
fHeaderChunk.putInt((blocksize / MIN_SIZE) * INT_SIZE, block);
fHeaderChunk.putInt((blocksize/BLOCK_SIZE_DELTA - MIN_BLOCK_DELTAS + 1) * INT_SIZE, block);
}
private void removeBlock(Chunk chunk, int blocksize, int block) throws CoreException {
assert fExclusiveLock;
int prevblock = chunk.getInt(block + PREV_OFFSET);
int nextblock = chunk.getInt(block + NEXT_OFFSET);
int prevblock = chunk.getInt(block + BLOCK_PREV_OFFSET);
int nextblock = chunk.getInt(block + BLOCK_NEXT_OFFSET);
if (prevblock != 0)
putInt(prevblock + NEXT_OFFSET, nextblock);
putInt(prevblock + BLOCK_NEXT_OFFSET, nextblock);
else // we were the head
setFirstBlock(blocksize, nextblock);
if (nextblock != 0)
putInt(nextblock + PREV_OFFSET, prevblock);
putInt(nextblock + BLOCK_PREV_OFFSET, prevblock);
}
private void addBlock(Chunk chunk, int blocksize, int block) throws CoreException {
assert fExclusiveLock;
// Mark our size
chunk.putInt(block, blocksize);
chunk.putShort(block, (short) blocksize);
// Add us to the head of the list
int prevfirst = getFirstBlock(blocksize);
chunk.putInt(block + PREV_OFFSET, 0);
chunk.putInt(block + NEXT_OFFSET, prevfirst);
chunk.putInt(block + BLOCK_PREV_OFFSET, 0);
chunk.putInt(block + BLOCK_NEXT_OFFSET, prevfirst);
if (prevfirst != 0)
putInt(prevfirst + PREV_OFFSET, block);
putInt(prevfirst + BLOCK_PREV_OFFSET, block);
setFirstBlock(blocksize, block);
}
@ -319,9 +324,9 @@ public class Database {
public void free(int offset) throws CoreException {
assert fExclusiveLock;
// TODO - look for opportunities to merge blocks
int block = offset - 4;
int block = offset - BLOCK_HEADER_SIZE;
Chunk chunk = getChunk(block);
int blocksize = - chunk.getInt(block);
int blocksize = - chunk.getShort(block);
if (blocksize < 0)
// already freed
throw new CoreException(new Status(IStatus.ERROR, CCorePlugin.PLUGIN_ID, 0, "Already Freed", new Exception())); //$NON-NLS-1$
@ -345,6 +350,14 @@ public class Database {
return getChunk(offset).getInt(offset);
}
public void put3ByteUnsignedInt(int offset, int value) throws CoreException {
getChunk(offset).put3ByteUnsignedInt(offset, value);
}
public int get3ByteUnsignedInt(int offset) throws CoreException {
return getChunk(offset).get3ByteUnsignedInt(offset);
}
public void putShort(int offset, short value) throws CoreException {
getChunk(offset).putShort(offset, value);
}
@ -400,12 +413,12 @@ public class Database {
System.out.println("free'd: " + freed); //$NON-NLS-1$
System.out.println("wasted: " + (fChunks.length * CHUNK_SIZE - (malloced - freed))); //$NON-NLS-1$
System.out.println("Free blocks"); //$NON-NLS-1$
for (int bs = MIN_SIZE; bs <= CHUNK_SIZE; bs += MIN_SIZE) {
for (int bs = MIN_BLOCK_DELTAS*BLOCK_SIZE_DELTA; bs <= CHUNK_SIZE; bs += BLOCK_SIZE_DELTA) {
int count = 0;
int block = getFirstBlock(bs);
while (block != 0) {
++count;
block = getInt(block + NEXT_OFFSET);
block = getInt(block + BLOCK_NEXT_OFFSET);
}
if (count != 0)
System.out.println("Block size: " + bs + "=" + count); //$NON-NLS-1$ //$NON-NLS-2$

View file

@ -36,13 +36,13 @@ public class LongString implements IString {
private static final int NEXT1 = 4;
private static final int CHARS1 = 8;
private static final int NUM_CHARS1 = (Database.MAX_SIZE - CHARS1) / 2;
private static final int NUM_CHARS1 = (Database.MAX_MALLOC_SIZE - CHARS1) / 2;
// Additional fields of subsequent records
private static final int NEXTN = 0;
private static final int CHARSN = 4;
private static final int NUM_CHARSN = (Database.MAX_SIZE - CHARSN) / 2;
private static final int NUM_CHARSN = (Database.MAX_MALLOC_SIZE - CHARSN) / 2;
public LongString(Database db, int record) {
this.db = db;
@ -55,7 +55,7 @@ public class LongString implements IString {
private int createString(int length, IWriter writer) throws CoreException {
// write the first record
int firstRecord = db.malloc(Database.MAX_SIZE);
int firstRecord = db.malloc(Database.MAX_MALLOC_SIZE);
int start = 0;
db.putInt(firstRecord, length);
writer.writeChars(start, NUM_CHARS1, firstRecord + CHARS1);
@ -64,7 +64,7 @@ public class LongString implements IString {
int lastNext = firstRecord + NEXT1;
start += NUM_CHARS1;
while (length - start > NUM_CHARSN) {
int nextRecord = db.malloc(Database.MAX_SIZE);
int nextRecord = db.malloc(Database.MAX_MALLOC_SIZE);
db.putInt(lastNext, nextRecord);
writer.writeChars(start, NUM_CHARSN, nextRecord + CHARSN);
start += NUM_CHARSN;

View file

@ -30,7 +30,7 @@ public class ShortString implements IString {
private static final int LENGTH = 0;
private static final int CHARS = 4;
public static final int MAX_LENGTH = (Database.MAX_SIZE - CHARS) / 2;
public static final int MAX_LENGTH = (Database.MAX_MALLOC_SIZE - CHARS) / 2;
public ShortString(Database db, int offset) {
this.db = db;

View file

@ -41,11 +41,11 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
private static final int BINDING_REC_OFFSET = 12;
private static final int BINDING_PREV_OFFSET = 16;
private static final int BINDING_NEXT_OFFSET = 20;
private static final int NODE_OFFSET_OFFSET = 24;
private static final int NODE_LENGTH_OFFSET = 28;
private static final int FLAGS = 32;
private static final int NODE_OFFSET_OFFSET = 24; // 3-byte unsigned int (sufficient for files <= 16mb)
private static final int NODE_LENGTH_OFFSET = 27; // short (sufficient for names <= 32k)
private static final int FLAGS = 29;
private static final int RECORD_SIZE = 36; // actual memory usage is the same from 28 - 44
private static final int RECORD_SIZE = 30; // 30 yields a 32-byte block. (31 would trigger a 40-byte block)
public static final int IS_DECLARATION = 1;
public static final int IS_DEFINITION = 2;
@ -71,7 +71,7 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
flags = IS_REFERENCE;
flags |= binding.getAdditionalNameFlags(flags, name);
db.putInt(record + FLAGS, flags);
db.putByte(record + FLAGS, (byte) flags);
// Hook us up to the binding
if (binding != null) {
@ -97,8 +97,8 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
// Record our location in the file
IASTFileLocation fileloc = name.getFileLocation();
db.putInt(record + NODE_OFFSET_OFFSET, fileloc.getNodeOffset());
db.putInt(record + NODE_LENGTH_OFFSET, fileloc.getNodeLength());
db.put3ByteUnsignedInt(record + NODE_OFFSET_OFFSET, fileloc.getNodeOffset());
db.putShort(record + NODE_LENGTH_OFFSET, (short) fileloc.getNodeLength());
}
public PDOMName(PDOM pdom, int nameRecord) {
@ -201,16 +201,16 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
}
private int getFlags(int mask) throws CoreException {
return pdom.getDB().getInt(record + FLAGS) & mask;
return pdom.getDB().getByte(record + FLAGS) & mask;
}
public void setIsBaseSpecifier(boolean val) throws CoreException {
int flags= pdom.getDB().getInt(record + FLAGS);
int flags= pdom.getDB().getByte(record + FLAGS) & 0xff;
if (val)
flags |= IS_INHERITANCE_SPEC;
else
flags &= ~IS_INHERITANCE_SPEC;
pdom.getDB().putInt(record + FLAGS, flags);
pdom.getDB().putByte(record + FLAGS, (byte) flags);
}
public boolean isBaseSpecifier() throws CoreException {
@ -288,7 +288,7 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
public int getNodeLength() {
try {
return pdom.getDB().getInt(record + NODE_LENGTH_OFFSET);
return (pdom.getDB().getShort(record + NODE_LENGTH_OFFSET)) & 0xffff;
} catch (CoreException e) {
CCorePlugin.log(e);
return 0;
@ -297,7 +297,7 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
public int getNodeOffset() {
try {
return pdom.getDB().getInt(record + NODE_OFFSET_OFFSET);
return pdom.getDB().get3ByteUnsignedInt(record + NODE_OFFSET_OFFSET);
} catch (CoreException e) {
CCorePlugin.log(e);
return 0;