mirrors/cdt
1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-07-24 17:35:35 +02:00
Code Issues Wiki Activity

Use prime numbers for hash table sizes to reduce collisions.

Browse source 
Change-Id: I4233e4a4ca729dd742825ee23b9c254fa836bc41
This commit is contained in:
Sergey Prigogin 2016-03-28 19:05:04 -07:00
parent eeaed8ee74
commit 9622166291
5 changed files with 269 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ObjectMapTest.java
View file

@ -14,6 +14,8 @@
*/
package org.eclipse.cdt.core.parser.tests;
import java.util.Random;
import org.eclipse.cdt.core.parser.util.CharArrayObjectMap;
import org.eclipse.cdt.core.parser.util.ObjectMap;
@ -24,7 +26,9 @@ import junit.framework.TestCase;
*/
public class ObjectMapTest extends TestCase {
static public class HashObject {
private static class HashObject {
final public int hash;
HashObject(int h) {
hash = h;
}
@ -33,7 +37,6 @@ public class ObjectMapTest extends TestCase {
public int hashCode() {
return hash;
}
final public int hash;
}
public void insertContents(ObjectMap map, Object[][] contents) throws Exception {
@ -60,7 +63,7 @@ public class ObjectMapTest extends TestCase {
assertContents(map, contents);
assertEquals(map.size(), 1);
assertEquals(map.capacity(), 2);
assertEquals(map.capacity(), 8);
}
public void testSimpleCollision() throws Exception{
@ -75,7 +78,7 @@ public class ObjectMapTest extends TestCase {
insertContents(map, contents);
assertEquals(map.size(), 2);
assertEquals(map.capacity(), 2);
assertEquals(map.capacity(), 8);
assertContents(map, contents);
}
@ -84,7 +87,7 @@ public class ObjectMapTest extends TestCase {
ObjectMap map = new ObjectMap(1);
assertEquals(map.size(), 0);
assertEquals(map.capacity(), 2);
assertEquals(map.capacity(), 8);
Object[][] res = new Object[][] { { "0", "o0" },
{ "1", "o1" },
@ -101,7 +104,7 @@ public class ObjectMapTest extends TestCase {
ObjectMap map = new ObjectMap(1);
assertEquals(map.size(), 0);
assertEquals(map.capacity(), 2);
assertEquals(map.capacity(), 8);
Object[][] res = new Object[][] { { new HashObject(0), "o0" },
{ new HashObject(1), "o1" },
@ -118,13 +121,13 @@ public class ObjectMapTest extends TestCase {
ObjectMap map = new ObjectMap(1);
assertEquals(map.size(), 0);
assertEquals(map.capacity(), 2);
assertEquals(map.capacity(), 8);
Object[][] res = new Object[][] { { "0", "o0" },
{ "1", "o1" } };
insertContents(map, res);
assertEquals(map.capacity(), 2);
assertEquals(map.capacity(), 8);
assertContents(map, res);
res = new Object[][]{ { "0", "o00" },
@ -136,21 +139,6 @@ public class ObjectMapTest extends TestCase {
assertContents(map, res);
}
public void testResizeResolvesCollision() throws Exception{
ObjectMap map = new ObjectMap(2);
Object k1 = new HashObject(0);
Object k2 = new HashObject(1);
Object k3 = new HashObject(4); // Collision with 0 in a table capacity 2, but ok in table capacity 4
Object[][] con = new Object[][] { { k1, "1" },
{ k2, "2" },
{ k3, "3" } };
insertContents(map, con);
assertContents(map, con);
}
public void testMapAdd() {
CharArrayObjectMap map = new CharArrayObjectMap(4);
char[] key1 = "key1".toCharArray();
@ -161,14 +149,25 @@ public class ObjectMapTest extends TestCase {
Object value2 = map.get(key2);
assertEquals(value1, value2);
for (int i = 0; i < 5; ++i) {
for (int i = 0; i < 25; ++i) {
map.put(("ikey" + i).toCharArray(), new Integer(i));
}
Object ivalue1 = map.get("ikey1".toCharArray());
assertEquals(ivalue1, new Integer(1));
for (int i = 0; i < 25; ++i) {
Object ivalue1 = map.get(("ikey" + i).toCharArray());
assertEquals(i, ivalue1);
}
}
Object ivalue4 = map.get("ikey4".toCharArray());
assertEquals(ivalue4, new Integer(4));
public void testCollisionRatio() {
Random random = new Random(239);
CharArrayObjectMap map = new CharArrayObjectMap(1);
for (int i = 0; i < 20000; i++) {
int r = random.nextInt();
map.put(("key" + Integer.toUnsignedString(i)).toCharArray(), i);
double collisionRatio = (double) map.countCollisions() / map.size();
assertTrue(String.format("Collision ratio %.3f is unexpectedly high for map size of %d.", collisionRatio, map.size()),
collisionRatio <= 0.4);
}
}
}

107
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/util/CharTable.java
View file

@ -25,7 +25,7 @@ public class CharTable extends HashTable {
super(initialSize);
keyTable = new char[capacity()][];
}
@Override
protected void resize(int size) {
char[][] oldKeyTable = keyTable;
@ -33,34 +33,34 @@ public class CharTable extends HashTable {
System.arraycopy(oldKeyTable, 0, keyTable, 0, Math.min(size, oldKeyTable.length));
super.resize(size);
}
@Override
public void clear() {
super.clear();
for (int i = 0; i < capacity(); i++)
keyTable[i] = null;
}
@Override
public Object clone() {
CharTable newTable = (CharTable) super.clone();
int size = capacity();
newTable.keyTable = new char[size][];
System.arraycopy(keyTable, 0, newTable.keyTable, 0, keyTable.length);
return newTable;
}
protected final int hash(char[] source, int start, int length) {
return CharArrayUtils.hash(source, start, length) & ((keyTable.length * 2) - 1);
return hashTable == null ? 0 : hashToOffset(CharArrayUtils.hash(source, start, length));
}
@Override
protected final int hash(int pos) {
return hash(keyTable[pos], 0, keyTable[pos].length);
}
protected final int hash(char[] obj) {
return hash(obj, 0, obj.length);
}
@ -68,52 +68,52 @@ public class CharTable extends HashTable {
protected final int addIndex(char[] buffer) {
return addIndex(buffer, 0, buffer.length);
}
public final int addIndex(char[] buffer, int start, int len) {
if (hashTable != null) {
int hash = hash(buffer, start, len);
int pos = lookup(buffer, start, len, hash);
if (pos != -1)
return pos;
// key is not here, add it.
if ((currEntry + 1) >= capacity()) {
resize();
hash = hash(buffer, start, len);
}
currEntry++;
keyTable[currEntry] = CharArrayUtils.extract(buffer, start, len);
linkIntoHashTable(currEntry, hash);
} else {
if (hashTable == null) {
int pos = lookup(buffer, start, len);
if (pos != -1)
return pos;
// key is not here, add it.
if ((currEntry + 1) >= capacity()) {
// Key is not here, add it.
if (currEntry + 1 >= capacity()) {
resize();
if (capacity() > minHashSize) {
//if we grew from list to hash, then recurse and add as a hashtable
if (hashTable != null) {
// If we grew from list to hash, then recurse and add as a hashtable.
return addIndex(buffer, start, len);
}
}
currEntry++;
keyTable[currEntry] = CharArrayUtils.extract(buffer, start, len);
} else {
int hash = hash(buffer, start, len);
int pos = lookup(buffer, start, len, hash);
if (pos != -1)
return pos;
// Key is not here, add it.
if (currEntry + 1 >= capacity()) {
resize();
hash = hash(buffer, start, len);
}
currEntry++;
keyTable[currEntry] = CharArrayUtils.extract(buffer, start, len);
linkIntoHashTable(currEntry, hash);
}
return currEntry;
}
protected void removeEntry(int i) {
// Remove the entry from the keyTable, shifting everything over if necessary
protected void removeEntry(int i) {
// Remove the entry from the keyTable, shifting everything over if necessary.
int hash = hash(keyTable[i]);
if (i < currEntry)
System.arraycopy(keyTable, i + 1, keyTable, i, currEntry - i);
System.arraycopy(keyTable, i + 1, keyTable, i, currEntry - i);
keyTable[currEntry] = null;
// Make sure you remove the value before calling super where currEntry will change
removeEntry(i, hash);
}
public List<char[]> toList() {
List<char[]> list = new ArrayList<char[]>(size());
int size = size();
@ -122,34 +122,34 @@ public class CharTable extends HashTable {
}
return list;
}
public final char[] keyAt(int i) {
if (i < 0 || i > currEntry)
return null;
return keyTable[i];
}
public final boolean containsKey(char[] key, int start, int len) {
return lookup(key, start, len) != -1;
}
public final boolean containsKey(char[] key) {
return lookup(key) != -1;
return lookup(key) != -1;
}
public final char[] findKey(char[] buffer, int start, int len) {
int idx = lookup(buffer, start, len);
if (idx == -1)
return null;
return keyTable[idx];
}
public int lookup(char[] buffer) {
return lookup(buffer, 0, buffer.length);
}
protected final int lookup(char[] buffer, int start, int len) {
if (hashTable != null)
return lookup(buffer, start, len, hash(buffer, start, len));
@ -159,24 +159,25 @@ public class CharTable extends HashTable {
}
return -1;
}
protected final int lookup(char[] buffer, int start, int len, int hash) {
if (hashTable[hash] == 0)
protected final int lookup(char[] buffer, int start, int len, int hash) {
int i = hashTable[hash];
if (i == 0)
return -1;
int i = hashTable[hash] - 1;
--i;
if (CharArrayUtils.equals(buffer, start, len, keyTable[i]))
return i;
// Follow the next chain
for (i = nextTable[i] - 1; i >= 0 && nextTable[i] != i + 1; i = nextTable[i] - 1) {
for (i = nextTable[i] - 1; i >= 0 && i != nextTable[i] - 1; i = nextTable[i] - 1) {
if (CharArrayUtils.equals(buffer, start, len, keyTable[i]))
return i;
}
return -1;
}
/**
* @since 5.7
*/

211
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/util/HashTable.java
View file

@ -10,40 +10,72 @@
*******************************************************************************/
package org.eclipse.cdt.core.parser.util;
import java.util.Arrays;
import java.util.Comparator;
/**
* @author ddaoust
*/
public class HashTable implements Cloneable {
protected static final int minHashSize = 2;
// Prime numbers from http://planetmath.org/goodhashtableprimes
private static final int[] PRIMES = {
17,
29,
53,
97,
193,
389,
769,
1543,
3079,
6151,
12289,
24593,
49157,
98317,
196613,
393241,
786433,
1572869,
3145739,
6291469,
12582917,
25165843,
50331653,
100663319,
201326611,
402653189,
805306457,
1610612741
};
private static final int MIN_HASH_SIZE = 9;
/** @deprecated Don't depend on this implementation detail. @noreference This field is not intended to be referenced by clients. */
@Deprecated
protected static final int minHashSize = MIN_HASH_SIZE - 1;
protected int currEntry = -1;
protected int[] hashTable;
protected int[] nextTable;
public boolean isEmpty() {
return currEntry < 0;
}
public final int size() {
return currEntry + 1;
}
public HashTable(int initialSize) {
int size = 1;
while (size < initialSize)
size <<= 1;
if (size > minHashSize) {
hashTable = new int[size * 2];
nextTable = new int[size];
if (initialSize >= MIN_HASH_SIZE) {
hashTable = new int[getSuitableHashTableSize(initialSize)];
nextTable = new int[initialSize];
} else {
hashTable = null;
nextTable = null;
}
}
@Override
public Object clone() {
HashTable newTable = null;
@ -53,11 +85,11 @@ public class HashTable implements Cloneable {
// Shouldn't happen because object supports clone.
return null;
}
int size = capacity();
if (hashTable != null) {
newTable.hashTable = new int[size * 2];
newTable.hashTable = new int[getSuitableHashTableSize(size)];
newTable.nextTable = new int[size];
System.arraycopy(hashTable, 0, newTable.hashTable, 0, hashTable.length);
System.arraycopy(nextTable, 0, newTable.nextTable, 0, nextTable.length);
@ -65,35 +97,29 @@ public class HashTable implements Cloneable {
newTable.currEntry = currEntry;
return newTable;
}
protected void resize() {
resize(capacity() << 1);
}
public void clear() {
currEntry = -1;
currEntry = -1;
// Clear the table.
if (hashTable == null)
return;
for (int i = 0; i < capacity(); i++) {
hashTable[2 * i] = 0;
hashTable[2 * i + 1] = 0;
nextTable[i] = 0;
}
Arrays.fill(hashTable, 0);
Arrays.fill(nextTable, 0);
}
protected void rehash() {
if (nextTable == null)
return;
// Clear the table (don't call clear() or else the subclasses stuff will be cleared too).
for (int i = 0; i < capacity(); i++) {
hashTable[2 * i] = 0;
hashTable[2 * i + 1] = 0;
nextTable[i] = 0;
}
Arrays.fill(hashTable, 0);
Arrays.fill(nextTable, 0);
// Need to rehash everything.
for (int i = 0; i <= currEntry; ++i) {
linkIntoHashTable(i, hash(i));
@ -101,77 +127,109 @@ public class HashTable implements Cloneable {
}
protected void resize(int size) {
if (size > minHashSize) {
hashTable = new int[size * 2];
if (size >= MIN_HASH_SIZE) {
hashTable = new int[getSuitableHashTableSize(size)];
nextTable = new int[size];
// Need to rehash everything.
for (int i = 0; i <= currEntry; ++i) {
linkIntoHashTable(i, hash(i));
}
}
}
private static int getSuitableHashTableSize(int size) {
size += (size + 2) / 3;
if (size < 0)
return Integer.MAX_VALUE; // Integer overflow. Return the max possible size.
int low = 0;
int high = PRIMES.length;
while (low < high) {
int mid = (low + high) >>> 1;
int p = PRIMES[mid];
if (p < size) {
low = mid + 1;
} else if (p > size) {
high = mid;
} else {
return p;
}
}
if (low == PRIMES.length)
return Integer.MAX_VALUE; // Largest prime is not sufficient. Return the max possible size.
return PRIMES[low];
}
protected int hash(int pos) {
// Return the hash value of the element in the key table.
// Return the hash value of the element in the key table.
throw new UnsupportedOperationException();
}
final int hashToOffset(int hash) {
int offset = hash % hashTable.length;
if (offset < 0)
offset += hashTable.length - 1;
return offset;
}
protected void linkIntoHashTable(int i, int hash) {
if (nextTable == null)
return;
if (hashTable[hash] == 0) {
hashTable[hash] = i + 1;
} else {
// Need to link.
int j = hashTable[hash] - 1;
while (nextTable[j] != 0) {
// if (nextTable[j] - 1 == j) {
// break;
// }
j = nextTable[j] - 1;
int k;
while ((k = nextTable[j]) != 0) {
j = k - 1;
}
nextTable[j] = i + 1;
}
}
public final int capacity() {
if (nextTable == null)
return minHashSize;
return MIN_HASH_SIZE - 1;
return nextTable.length;
}
protected void removeEntry(int i, int hash) {
if (nextTable == null) {
--currEntry;
return;
}
// Remove the hash entry.
if (hashTable[hash] == i + 1) {
hashTable[hash] = nextTable[i];
} else {
} else {
// Find entry pointing to me.
int j = hashTable[hash] - 1;
while (nextTable[j] != 0 && nextTable[j] != i + 1)
j = nextTable[j] - 1;
int k;
while ((k = nextTable[j]) != 0 && k != i + 1) {
j = k - 1;
}
nextTable[j] = nextTable[i];
}
if (i < currEntry) {
// Shift everything over.
System.arraycopy(nextTable, i + 1, nextTable, i, currEntry - i);
// Adjust hash and next entries for things that moved.
for (int j = 0; j < hashTable.length; ++j) {
if (hashTable[j] > i + 1)
--hashTable[j];
int k = hashTable[j] - 1;
if (k > i)
hashTable[j] = k;
}
for (int j = 0; j < nextTable.length; ++j) {
if (nextTable[j] > i + 1)
--nextTable[j];
int k = nextTable[j] - 1;
if (k > i)
nextTable[j] = k;
}
}
@ -182,37 +240,60 @@ public class HashTable implements Cloneable {
public final void sort(Comparator<Object> c) {
if (size() > 1) {
quickSort(c, 0, size() - 1);
quickSort(c, 0, size() - 1);
rehash();
}
}
}
private void quickSort(Comparator<Object> c, int p, int r) {
if (p < r) {
int q = partition(c, p, r);
if (p < q) quickSort(c, p, q);
if (++q < r) quickSort(c, q, r);
if (p < q)
quickSort(c, p, q);
if (++q < r)
quickSort(c, q, r);
}
}
protected int partition(Comparator<Object> c, int p, int r) {
throw new UnsupportedOperationException();
}
/**
* For debugging only.
* @noreference This method is not intended to be referenced by clients.
*/
public void dumpNexts() {
if (nextTable == null)
return;
for (int i = 0; i < nextTable.length; ++i) {
if (nextTable[i] == 0)
continue;
System.out.print(i);
for (int j = nextTable[i] - 1; j >= 0; j = nextTable[j] - 1)
System.out.print(" -> " + j); //$NON-NLS-1$
System.out.println(""); //$NON-NLS-1$
}
}
/**
* Returns the number of collisions.
* For debugging only.
* @noreference This method is not intended to be referenced by clients.
*/
public int countCollisions() {
if (nextTable == null)
return 0;
int numCollisions = 0;
for (int i = 0; i < nextTable.length; ++i) {
if (nextTable[i] != 0)
numCollisions++;
}
return numCollisions;
}
}

26
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/util/ObjectMap.java
View file

@ -6,7 +6,7 @@
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Andrew Niefer (IBM Corporation) - Initial API and implementation
* Andrew Niefer (IBM Corporation) - Initial API and implementation
*******************************************************************************/
package org.eclipse.cdt.core.parser.util;
@ -24,7 +24,7 @@ public class ObjectMap extends ObjectTable<Object> {
@Override
public Object put(Object key, Object value) { throw new UnsupportedOperationException(); }
};
private Object[] valueTable;
public ObjectMap(int initialSize) {
@ -39,13 +39,13 @@ public class ObjectMap extends ObjectTable<Object> {
System.arraycopy(valueTable, 0, newMap.valueTable, 0, valueTable.length);
return newMap;
}
@Override
final public void clear() {
super.clear();
Arrays.fill(valueTable, null);
}
@Override
protected void resize(int size) {
Object[] oldValueTable = valueTable;
@ -53,7 +53,7 @@ public class ObjectMap extends ObjectTable<Object> {
System.arraycopy(oldValueTable, 0, valueTable, 0, oldValueTable.length);
super.resize(size);
}
public Object put(Object key, Object value) {
int i = add(key);
Object oldvalue = valueTable[i];
@ -71,10 +71,10 @@ public class ObjectMap extends ObjectTable<Object> {
final public Object getAt(int i) {
if (i < 0 || i > currEntry)
return null;
return valueTable[i];
}
final public Object remove(Object key) {
if (key == null)
return null;
@ -84,10 +84,10 @@ public class ObjectMap extends ObjectTable<Object> {
Object value = valueTable[i];
removeEntry(i);
return value;
}
@Override
final protected void removeEntry(int i) {
// Remove the entry from the valueTable, shifting everything over if necessary
@ -98,14 +98,14 @@ public class ObjectMap extends ObjectTable<Object> {
// Make sure you remove the value before calling super where currEntry will change
super.removeEntry(i);
}
@Override
protected int partition(Comparator<Object> c, int p, int r) {
Object x = keyTable[p];
Object temp = null;
int i = p;
int j = r;
while (true) {
while (c.compare(keyTable[j], x) > 0) {
j--;
@ -115,12 +115,12 @@ public class ObjectMap extends ObjectTable<Object> {
i++;
}
}
if (i < j) {
temp = keyTable[j];
keyTable[j] = keyTable[i];
keyTable[i] = temp;
temp = valueTable[j];
valueTable[j] = valueTable[i];
valueTable[i] = temp;

58
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/util/ObjectTable.java
View file

@ -18,7 +18,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
public abstract class ObjectTable<T> extends HashTable implements Iterable<T> {
public abstract class ObjectTable<T> extends HashTable implements Iterable<T> {
protected T[] keyTable;
@SuppressWarnings("unchecked")
@ -31,14 +31,14 @@ public abstract class ObjectTable<T> extends HashTable implements Iterable<T> {
@SuppressWarnings("unchecked")
public Object clone() {
ObjectTable<T> newTable = (ObjectTable<T>) super.clone();
int size = capacity();
newTable.keyTable = (T[]) new Object[size];
System.arraycopy(keyTable, 0, newTable.keyTable, 0, keyTable.length);
return newTable;
}
public List<T> toList() {
int size = size();
List<T> list = new ArrayList<T>(size);
@ -51,26 +51,26 @@ public abstract class ObjectTable<T> extends HashTable implements Iterable<T> {
public T keyAt(int i) {
if (i < 0 || i > currEntry)
return null;
return keyTable[i];
}
@Override
public void clear() {
super.clear();
for (int i = 0; i < keyTable.length; i++)
keyTable[i] = null;
}
@Override
protected final int hash(int pos) {
return hash(keyTable[pos]);
}
private int hash(Object obj) {
return obj.hashCode() & ((capacity() * 2) - 1);
return hashTable == null ? 0 : hashToOffset(obj.hashCode());
}
@Override
@SuppressWarnings("unchecked")
protected void resize(int size) {
@ -79,13 +79,13 @@ public abstract class ObjectTable<T> extends HashTable implements Iterable<T> {
System.arraycopy(oldKeyTable, 0, keyTable, 0, oldKeyTable.length);
super.resize(size);
}
protected final int add(T obj) {
int pos = lookup(obj);
if (pos != -1)
return pos;
if ((currEntry + 1) >= capacity()) {
if (currEntry + 1 >= capacity()) {
resize();
}
currEntry++;
@ -93,55 +93,55 @@ public abstract class ObjectTable<T> extends HashTable implements Iterable<T> {
linkIntoHashTable(currEntry, hash(obj));
return currEntry;
}
protected void removeEntry(int i) {
protected void removeEntry(int i) {
// Remove the entry from the keyTable, shifting everything over if necessary
int hash = hash(keyTable[i]);
if (i < currEntry)
System.arraycopy(keyTable, i + 1, keyTable, i, currEntry - i);
System.arraycopy(keyTable, i + 1, keyTable, i, currEntry - i);
keyTable[currEntry] = null;
// Make sure you remove the value before calling super where currEntry will change
removeEntry(i, hash);
}
protected final int lookup(Object buffer) {
if (hashTable != null) {
int hash = hash(buffer);
if (hashTable[hash] == 0)
return -1;
int i = hashTable[hash] - 1;
if (buffer.equals(keyTable[i]))
return i;
// Follow the next chain
for (i = nextTable[i] - 1; i >= 0 && nextTable[i] != i + 1; i = nextTable[i] - 1) {
for (i = nextTable[i] - 1; i >= 0 && i != nextTable[i] - 1; i = nextTable[i] - 1) {
if (buffer.equals(keyTable[i]))
return i;
}
return -1;
}
for (int i = 0; i <= currEntry; i++) {
if (buffer.equals(keyTable[i]))
return i;
}
return -1;
return -1;
}
public boolean containsKey(T key) {
return lookup(key) != -1;
return lookup(key) != -1;
}
public Object[] keyArray() {
Object[] keys = new Object[size()];
System.arraycopy(keyTable, 0, keys, 0, keys.length);
return keys;
}
@SuppressWarnings("unchecked")
public <X> X[] keyArray(Class<X> c) {
X[] keys = (X[]) Array.newInstance(c, size());
@ -153,7 +153,7 @@ public abstract class ObjectTable<T> extends HashTable implements Iterable<T> {
if (size() != other.size()) {
return false;
}
for (int i = 0; i < keyTable.length; i++) {
T key1 = keyTable[i];
T key2 = other.keyTable[i];