mirror of
https://github.com/eclipse-cdt/cdt
synced 2025-07-04 23:55:26 +02:00
[dstore] fix issues with the spiriting mechanism and other memory
improvements (phase 1)
This commit is contained in:
David McKnight
parent
d7b4eb0e9b
commit
849647cef6
8 changed files with 312 additions and 308 deletions
|
|
@ -19,6 +19,7 @@
|
|||
* David McKnight (IBM) - [389286] [dstore] element delete should not clear _attributes since elements get recycled
|
||||
* David McKnight (IBM) - [390037] [dstore] Duplicated items in the System view
|
||||
* David McKnight (IBM) - [391065] [dstore][regression] DataElement with "" type treated as deleted (when it's not)
|
||||
* David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
|
||||
*******************************************************************************/
|
||||
|
||||
package org.eclipse.dstore.core.model;
|
||||
|
|
@ -28,7 +29,6 @@ import java.util.List;
|
|||
|
||||
import org.eclipse.dstore.core.util.StringCompare;
|
||||
import org.eclipse.dstore.extra.IDataElement;
|
||||
import org.eclipse.dstore.internal.core.util.DataElementRemover;
|
||||
import org.eclipse.dstore.internal.extra.DataElementActionFilter;
|
||||
import org.eclipse.dstore.internal.extra.DesktopElement;
|
||||
import org.eclipse.dstore.internal.extra.PropertySource;
|
||||
|
|
@ -76,7 +76,6 @@ public final class DataElement implements IDataElement
|
|||
{
|
||||
_dataStore = null;
|
||||
_parent = null;
|
||||
DataElementRemover.addToCreatedCount();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -88,7 +87,6 @@ public final class DataElement implements IDataElement
|
|||
{
|
||||
_dataStore = dataStore;
|
||||
_parent = null;
|
||||
DataElementRemover.addToCreatedCount();
|
||||
}
|
||||
|
||||
/* Apparently having this method causes the GC to delay
|
||||
|
|
@ -304,6 +302,7 @@ public final class DataElement implements IDataElement
|
|||
public void reInitAsTransient(String attributes[])
|
||||
{
|
||||
_attributes = attributes;
|
||||
setAttribute(DE.A_ID, DataStoreResources.model_transient);
|
||||
|
||||
_isReference = false;
|
||||
_isDescriptor = false;
|
||||
|
|
@ -886,14 +885,19 @@ public final class DataElement implements IDataElement
|
|||
*/
|
||||
public void setSpirit(boolean flag)
|
||||
{
|
||||
_isSpirit = flag;
|
||||
String refType = getAttribute(DE.A_REF_TYPE);
|
||||
if (refType != null){
|
||||
if (_isSpirit && !refType.equals(DataStoreResources.SPIRIT)) {
|
||||
setAttribute(DE.A_REF_TYPE, DataStoreResources.SPIRIT);
|
||||
}
|
||||
else if (refType.equals(DataStoreResources.SPIRIT)){ // if it was a spirit, change it back
|
||||
setAttribute(DE.A_REF_TYPE, DataStoreResources.VALUE);
|
||||
if (flag && isDescriptor()){
|
||||
// descriptors should not be spirited
|
||||
}
|
||||
else {
|
||||
_isSpirit = flag;
|
||||
String refType = getAttribute(DE.A_REF_TYPE);
|
||||
if (refType != null){
|
||||
if (_isSpirit && !refType.equals(DataStoreResources.SPIRIT)) {
|
||||
setAttribute(DE.A_REF_TYPE, DataStoreResources.SPIRIT);
|
||||
}
|
||||
else if (refType.equals(DataStoreResources.SPIRIT)){ // if it was a spirit, change it back
|
||||
setAttribute(DE.A_REF_TYPE, DataStoreResources.VALUE);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1454,6 +1458,8 @@ public final class DataElement implements IDataElement
|
|||
+ getSource()
|
||||
+ "\n\tDepth:\t" //$NON-NLS-1$
|
||||
+ _depth
|
||||
+ "\n\tisSpirit:\t" //$NON-NLS-1$
|
||||
+ _isSpirit
|
||||
+ "\n\tDataStore:\t" //$NON-NLS-1$
|
||||
+ _dataStore.getName()
|
||||
+ "\n}\n"; //$NON-NLS-1$
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@
|
|||
* David McKnight (IBM) - [385097] [dstore] DataStore spirit mechanism is not enabled
|
||||
* David McKnight (IBM) - [385793] [dstore] DataStore spirit mechanism and other memory improvements needed
|
||||
* David McKnight (IBM) - [390037] [dstore] Duplicated items in the System view
|
||||
* David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
|
||||
*******************************************************************************/
|
||||
|
||||
package org.eclipse.dstore.core.model;
|
||||
|
|
@ -199,7 +200,7 @@ public final class DataStore
|
|||
*
|
||||
* This needs to be updated for each major release.
|
||||
*/
|
||||
private String _RSE_version = "3.3.1"; //$NON-NLS-1$
|
||||
private String _RSE_version = "3.4.2"; //$NON-NLS-1$
|
||||
|
||||
/**
|
||||
* Creates a new <code>DataStore</code> instance
|
||||
|
|
@ -950,10 +951,6 @@ public final class DataStore
|
|||
reference.reInit(parent, realObject, relationType);
|
||||
parent.addNestedData(reference, false);
|
||||
|
||||
String sugId = reference.getId();
|
||||
synchronized (_hashMap){
|
||||
_hashMap.put(sugId, reference);
|
||||
}
|
||||
refresh(parent);
|
||||
|
||||
return reference;
|
||||
|
|
@ -1005,11 +1002,6 @@ public final class DataStore
|
|||
|
||||
parent.addNestedData(reference, false);
|
||||
|
||||
String sugId = reference.getId();
|
||||
synchronized (_hashMap){
|
||||
_hashMap.put(sugId, reference);
|
||||
}
|
||||
|
||||
if (doRefresh)
|
||||
{
|
||||
refresh(parent);
|
||||
|
|
@ -1078,22 +1070,12 @@ public final class DataStore
|
|||
toReference.reInit(parent, realObject, toRelation);
|
||||
|
||||
parent.addNestedData(toReference, false);
|
||||
|
||||
String toId = toReference.getId();
|
||||
synchronized (_hashMap){
|
||||
_hashMap.put(toId, toReference);
|
||||
}
|
||||
// reference with "from" relationship
|
||||
DataElement fromReference = createElement();
|
||||
|
||||
fromReference.reInit(realObject, parent, fromRelation);
|
||||
|
||||
realObject.addNestedData(fromReference, false);
|
||||
|
||||
String fromId = fromReference.getId();
|
||||
synchronized (_hashMap){
|
||||
_hashMap.put(fromId, fromReference);
|
||||
}
|
||||
refresh(parent);
|
||||
|
||||
|
||||
|
|
@ -1128,12 +1110,6 @@ public final class DataStore
|
|||
}
|
||||
|
||||
parent.addNestedData(toReference, false);
|
||||
|
||||
String toId = toReference.getId();
|
||||
synchronized (_hashMap){
|
||||
_hashMap.put(toId, toReference);
|
||||
}
|
||||
|
||||
// reference with "from" relationship
|
||||
DataElement fromReference = createElement();
|
||||
|
||||
|
|
@ -1149,10 +1125,6 @@ public final class DataStore
|
|||
|
||||
realObject.addNestedData(fromReference, false);
|
||||
|
||||
String fromId = fromReference.getId();
|
||||
synchronized (_hashMap){
|
||||
_hashMap.put(fromId, fromReference);
|
||||
}
|
||||
refresh(parent);
|
||||
|
||||
|
||||
|
|
@ -1207,7 +1179,7 @@ public final class DataStore
|
|||
|
||||
public DataElement createTransientObject(String attributes[])
|
||||
{
|
||||
DataElement newObject = createElement();
|
||||
DataElement newObject = new DataElement(this);
|
||||
|
||||
newObject.reInitAsTransient(attributes);
|
||||
return newObject;
|
||||
|
|
@ -2537,7 +2509,7 @@ public final class DataStore
|
|||
*/
|
||||
public DataElement command(DataElement commandDescriptor, ArrayList arguments, DataElement dataObject)
|
||||
{
|
||||
return command(commandDescriptor, arguments, dataObject, false);
|
||||
return command(commandDescriptor, arguments, dataObject, true);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -2553,7 +2525,8 @@ public final class DataStore
|
|||
{
|
||||
if (_commandHandler != null)
|
||||
{
|
||||
return _commandHandler.command(commandDescriptor, arguments, dataObject, true, immediate);
|
||||
// as per bug #396440, default is now to not use references
|
||||
return _commandHandler.command(commandDescriptor, arguments, dataObject, false, immediate);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
|
@ -2584,7 +2557,8 @@ public final class DataStore
|
|||
{
|
||||
if (_commandHandler != null)
|
||||
{
|
||||
return _commandHandler.command(commandDescriptor, arg, dataObject, true, immediate);
|
||||
// as per bug #396440, default is now to not use references
|
||||
return _commandHandler.command(commandDescriptor, arg, dataObject, false, immediate);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
|
@ -2598,6 +2572,7 @@ public final class DataStore
|
|||
*/
|
||||
public DataElement command(DataElement commandDescriptor, DataElement dataObject)
|
||||
{
|
||||
// as per bug #396440, default is now to not use references
|
||||
return command(commandDescriptor, dataObject, false);
|
||||
}
|
||||
|
||||
|
|
@ -2888,31 +2863,19 @@ public final class DataStore
|
|||
|
||||
|
||||
List searchList = root.getNestedData();
|
||||
|
||||
if (searchList != null)
|
||||
{
|
||||
for (int i = 0; i < searchList.size(); i++)
|
||||
{
|
||||
if (searchList != null){
|
||||
for (int i = 0; i < searchList.size(); i++){
|
||||
DataElement child = (DataElement) searchList.get(i);
|
||||
if (child != null)
|
||||
{
|
||||
synchronized (child)
|
||||
{
|
||||
if (child.isDeleted() && !results.contains(child))
|
||||
{
|
||||
results.add(child);
|
||||
if (!child.isReference())
|
||||
{
|
||||
if (depth > 0)
|
||||
{
|
||||
List sResults = findDeleted(child, depth - 1);
|
||||
for (int j = 0; j < sResults.size(); j++)
|
||||
{
|
||||
results.add(sResults.get(j));
|
||||
}
|
||||
if (child != null){
|
||||
if (child.isDeleted() && !results.contains(child)){
|
||||
results.add(child);
|
||||
if (!child.isReference()){
|
||||
if (depth > 0){
|
||||
List sResults = findDeleted(child, depth - 1);
|
||||
for (int j = 0; j < sResults.size(); j++){
|
||||
results.add(sResults.get(j));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3983,16 +3946,12 @@ public final class DataStore
|
|||
|
||||
int numRecycled = _recycled.size();
|
||||
|
||||
if (numRecycled > 1)
|
||||
{
|
||||
synchronized (_recycled)
|
||||
{
|
||||
if (numRecycled > _MAX_FREE)
|
||||
{
|
||||
if (numRecycled > 0){
|
||||
synchronized (_recycled){
|
||||
if (numRecycled > _MAX_FREE){
|
||||
int numRemoved = numRecycled - _MAX_FREE;
|
||||
for (int i = 1; i <= numRemoved; i++)
|
||||
{
|
||||
_recycled.remove(numRemoved - i);
|
||||
for (int i = numRemoved - 1; i >=0; i--){
|
||||
_recycled.remove(i);
|
||||
}
|
||||
}
|
||||
newObject = (DataElement) _recycled.remove((_recycled.size() - 1));
|
||||
|
|
@ -4076,16 +4035,16 @@ public final class DataStore
|
|||
|
||||
private void disconnectObjectHelper(DataElement toDisconnect, int depth)
|
||||
{
|
||||
if (depth > 0)
|
||||
{
|
||||
if (depth > 0){
|
||||
depth--;
|
||||
_deRemover.addToQueueForRemoval(toDisconnect);
|
||||
for (int i = 0; i < toDisconnect.getNestedSize(); i++)
|
||||
{
|
||||
DataElement subDisconnect = toDisconnect.get(i);
|
||||
if (subDisconnect != null && subDisconnect.getDataStore() == this && !subDisconnect.isSpirit())
|
||||
{
|
||||
disconnectObjectHelper(subDisconnect, depth);
|
||||
List nestedData = toDisconnect.getNestedData();
|
||||
if (nestedData != null){
|
||||
for (int i = 0; i < nestedData.size(); i++){
|
||||
DataElement subDisconnect = (DataElement)nestedData.get(i);
|
||||
if (subDisconnect != null && !subDisconnect.isSpirit() && !subDisconnect.isDescriptor() && !subDisconnect.isReference()){
|
||||
disconnectObjectHelper(subDisconnect, depth);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -4105,18 +4064,8 @@ public final class DataStore
|
|||
else
|
||||
{
|
||||
return generateId();
|
||||
/*
|
||||
String newId = String.valueOf(_random.nextInt());
|
||||
while (_hashMap.containsKey(newId))
|
||||
{
|
||||
newId = String.valueOf(_random.nextInt());
|
||||
}
|
||||
|
||||
return newId;
|
||||
*/
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
private String generateId(DataElement parent, String type, String name)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
* Contributors:
|
||||
* David McKnight (IBM) - [226561] [apidoc] Add API markup to RSE Javadocs where extend / implement is allowed
|
||||
* David McKnight (IBM) - [385097] [dstore] DataStore spirit mechanism is not enabled
|
||||
* David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
|
||||
********************************************************************************/
|
||||
|
||||
package org.eclipse.dstore.core.model;
|
||||
|
|
@ -191,9 +192,6 @@ public class DataStoreSchema
|
|||
// miner-specific descriptors are defined in the miners when they extend the schema
|
||||
|
||||
// these first elements are the most fundamental
|
||||
// DataElement uiCmdD = _dataStore.createObject(schemaRoot, DE.T_UI_COMMAND_DESCRIPTOR, DE.T_UI_COMMAND_DESCRIPTOR);
|
||||
_dataStore.createObject(schemaRoot, DE.T_UI_COMMAND_DESCRIPTOR, DE.T_UI_COMMAND_DESCRIPTOR);
|
||||
|
||||
_commandDescriptor = _dataStore.createCommandDescriptor(schemaRoot, DE.T_COMMAND_DESCRIPTOR);
|
||||
_objectDescriptor = _dataStore.createObjectDescriptor(schemaRoot, DE.T_OBJECT_DESCRIPTOR);
|
||||
_relationDescriptor = _dataStore.createRelationDescriptor(schemaRoot, DE.T_RELATION_DESCRIPTOR);
|
||||
|
|
|
|||
|
|
@ -17,15 +17,16 @@
|
|||
* David McKnight (IBM) - [385793] [dstore] DataStore spirit mechanism and other memory improvements needed
|
||||
* David McKnight (IBM) - [389286] [dstore] element delete should not clear _attributes since elements get recycled
|
||||
* David McKnight (IBM) - [390037] [dstore] Duplicated items in the System view
|
||||
* David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
|
||||
*******************************************************************************/
|
||||
|
||||
package org.eclipse.dstore.core.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import org.eclipse.dstore.core.java.IRemoteClassInstance;
|
||||
import org.eclipse.dstore.internal.core.util.DataElementRemover;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
|
|
@ -75,49 +76,83 @@ public abstract class UpdateHandler extends Handler
|
|||
clean(object, 2);
|
||||
}
|
||||
|
||||
|
||||
private void unmap(DataElement element)
|
||||
{
|
||||
if (element.isDeleted() || element.isSpirit()){
|
||||
HashMap map = _dataStore.getHashMap();
|
||||
synchronized (map){
|
||||
String id = element.getId();
|
||||
// _dataStore.memLog("unmapping " + id);
|
||||
map.remove(id);
|
||||
_dataStore.addToRecycled(element);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void clean(DataElement object, int depth)
|
||||
{
|
||||
if ((depth > 0) && (object != null) && object.getNestedSize() > 0)
|
||||
{
|
||||
List deletedList = _dataStore.findDeleted(object);
|
||||
boolean isServer = !_dataStore.isVirtual();
|
||||
if ((depth > 0) && object != null){
|
||||
if (object.isSpirit() || object.isDeleted()){
|
||||
DataElement parent = object.getParent();
|
||||
cleanChildren(object);
|
||||
unmap(object);
|
||||
if (object.isSpirit() && isServer){
|
||||
// officially delete this now
|
||||
object.delete();
|
||||
}
|
||||
if (isServer){
|
||||
object.clear();
|
||||
}
|
||||
if (parent != null){
|
||||
synchronized (parent){
|
||||
parent.removeNestedData(object);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (object.getNestedSize() > 0){
|
||||
cleanChildren(object);
|
||||
}
|
||||
}
|
||||
// remove objects under temproot
|
||||
DataElement tmpRoot = _dataStore.getTempRoot();
|
||||
if (tmpRoot != null){
|
||||
tmpRoot.removeNestedData();
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < deletedList.size(); i++)
|
||||
{
|
||||
|
||||
/*
|
||||
protected void clean(DataElement object, int depth)
|
||||
{
|
||||
boolean isServer = !_dataStore.isVirtual();
|
||||
if ((depth > 0) && object != null){
|
||||
List deletedList = _dataStore.findDeleted(object);
|
||||
for (int i = 0; i < deletedList.size(); i++){
|
||||
DataElement child = (DataElement) deletedList.get(i);
|
||||
if (child != null && child.isDeleted())
|
||||
{
|
||||
if (child != null && child.isDeleted()){
|
||||
DataElement parent = child.getParent();
|
||||
DataElementRemover.addToRemovedCount();
|
||||
|
||||
cleanChildren(child); // clean the children
|
||||
|
||||
|
||||
if (child.isSpirit())
|
||||
{
|
||||
unmap(child);
|
||||
if (child.isSpirit() && isServer){
|
||||
// officially delete this now
|
||||
// will only happen on server since, on client,
|
||||
// the above call to isDeleted() returns false for spirited
|
||||
child.delete();
|
||||
}
|
||||
|
||||
child.clear();
|
||||
if (parent != null)
|
||||
{
|
||||
synchronized (parent)
|
||||
{
|
||||
if (parent != null){
|
||||
synchronized (parent){
|
||||
parent.removeNestedData(child);
|
||||
}
|
||||
}
|
||||
// _dataStore.addToRecycled(child);
|
||||
}
|
||||
}
|
||||
|
||||
deletedList.clear();
|
||||
}
|
||||
// delete objects under temproot
|
||||
_dataStore.getTempRoot().removeNestedData();
|
||||
|
||||
}
|
||||
*/
|
||||
|
||||
/**
|
||||
* Recursively clean children for deletion
|
||||
|
|
@ -125,25 +160,26 @@ public abstract class UpdateHandler extends Handler
|
|||
*/
|
||||
protected void cleanChildren(DataElement parent)
|
||||
{
|
||||
boolean isServer = !_dataStore.isVirtual();
|
||||
List nestedData = parent.getNestedData();
|
||||
if (nestedData != null)
|
||||
{
|
||||
if (nestedData != null){
|
||||
synchronized (nestedData){
|
||||
for (int i = nestedData.size() - 1; i >= 0; i--){
|
||||
DataElement child = (DataElement)nestedData.get(i);
|
||||
cleanChildren(child);
|
||||
|
||||
if (child.isSpirit())
|
||||
{
|
||||
// officially delete this now
|
||||
child.delete();
|
||||
if (child.isSpirit() || child.isDeleted()){
|
||||
cleanChildren(child);
|
||||
unmap(child);
|
||||
|
||||
if (isServer){
|
||||
// officially delete this now
|
||||
child.delete();
|
||||
child.clear();
|
||||
}
|
||||
nestedData.remove(child);
|
||||
}
|
||||
|
||||
child.clear();
|
||||
parent.removeNestedData(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -172,51 +208,39 @@ public abstract class UpdateHandler extends Handler
|
|||
* @param object an object to get updated
|
||||
* @param immediate true indicates that this object should be first in the queue
|
||||
*/
|
||||
public void update(DataElement object, boolean immediate)
|
||||
{
|
||||
synchronized (_dataObjects)
|
||||
{
|
||||
if (object != null){
|
||||
String type = object.getType();
|
||||
boolean isStatus = DataStoreResources.model_status.equals(type);
|
||||
if (immediate)
|
||||
{
|
||||
_dataObjects.add(0, object);
|
||||
// none of this immediate stuff - just put it at the beginning
|
||||
//handle();
|
||||
public void update(DataElement object, boolean immediate){
|
||||
if (object != null){
|
||||
String type = object.getType();
|
||||
boolean statusDone = false;
|
||||
boolean isStatus = DataStoreResources.model_status.equals(type);
|
||||
if (isStatus){
|
||||
statusDone = DataStoreResources.model_done.equals(object.getName()) || DataStoreResources.model_done.equals(object.getValue());
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!_dataObjects.contains(object))
|
||||
{
|
||||
_dataObjects.add(object);
|
||||
synchronized (_dataObjects){
|
||||
if (immediate){
|
||||
_dataObjects.add(0, object);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
if (_dataStore != null && object != null && !object.isDeleted())
|
||||
{
|
||||
if (isStatus)
|
||||
{
|
||||
if (object.getName().equals(DataStoreResources.model_done))
|
||||
{
|
||||
//DKM
|
||||
else {
|
||||
if (!_dataObjects.contains(object)){
|
||||
_dataObjects.add(object);
|
||||
}
|
||||
else {
|
||||
if (_dataStore != null && object != null && !object.isDeleted()){
|
||||
if (isStatus && statusDone){
|
||||
// move to the back of the queue
|
||||
// this is done so that if status that was already queued changed to done in between
|
||||
// requests, and had not yet been transferred over comm layer, the completed status
|
||||
// object does not come back to client (as "done") before the results of a query
|
||||
_dataObjects.remove(object);
|
||||
_dataObjects.add(object);
|
||||
|
||||
_dataObjects.add(object);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (_dataStore != null && !_dataStore.isVirtual() && isStatus){
|
||||
_dataStore.disconnectObjects(object); // spirit the status
|
||||
}
|
||||
|
||||
if (_dataStore != null && !_dataStore.isVirtual() && isStatus && statusDone){
|
||||
_dataStore.disconnectObject(object.getParent()); // spirit the command and its children
|
||||
// _dataStore.disconnectObject(object);
|
||||
}
|
||||
}
|
||||
notifyInput();
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
* David McKnight (IBM) - [225507][api][breaking] RSE dstore API leaks non-API types
|
||||
* David McKnight (IBM) - [226561] [apidoc] Add API markup to RSE Javadocs where extend / implement is allowed
|
||||
* David McKnight (IBM) - [390037] [dstore] Duplicated items in the System view
|
||||
* David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
|
||||
*******************************************************************************/
|
||||
|
||||
package org.eclipse.dstore.core.util;
|
||||
|
|
@ -43,7 +44,6 @@ public class CommandGenerator
|
|||
private DataStore _dataStore = null;
|
||||
private DataElement _log = null;
|
||||
|
||||
static private int _id = 0;
|
||||
|
||||
/**
|
||||
* Constructor
|
||||
|
|
@ -143,7 +143,7 @@ public class CommandGenerator
|
|||
*/
|
||||
public DataElement generateCommand(DataElement commandDescriptor, ArrayList arguments, DataElement dataObject, boolean refArg)
|
||||
{
|
||||
|
||||
//refArg = false;
|
||||
DataElement commandObject = createCommand(commandDescriptor);
|
||||
if (commandObject != null)
|
||||
{
|
||||
|
|
@ -151,7 +151,7 @@ public class CommandGenerator
|
|||
|
||||
commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName());
|
||||
|
||||
if (dataObject.isUpdated() && !dataObject.isSpirit())
|
||||
if (refArg && !dataObject.isSpirit())
|
||||
{
|
||||
_dataStore.createReference(commandObject, dataObject,DataStoreResources.model_contents);
|
||||
}
|
||||
|
|
@ -175,7 +175,7 @@ public class CommandGenerator
|
|||
DataElement arg = (DataElement) arguments.get(i);
|
||||
if (arg != null)
|
||||
{
|
||||
if (!arg.isUpdated() || arg.isSpirit())
|
||||
if (!arg.isUpdated() || arg.isSpirit() || !refArg)
|
||||
{
|
||||
commandObject.addNestedData(arg, false);
|
||||
}
|
||||
|
|
@ -207,14 +207,14 @@ public class CommandGenerator
|
|||
*/
|
||||
public DataElement generateCommand(DataElement commandDescriptor, DataElement arg, DataElement dataObject, boolean refArg)
|
||||
{
|
||||
_id++;
|
||||
//refArg = false;
|
||||
|
||||
DataElement commandObject = createCommand(commandDescriptor);
|
||||
if (commandObject != null)
|
||||
{
|
||||
commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName());
|
||||
clearDeleted(dataObject);
|
||||
if ((refArg || dataObject.isUpdated()) && !dataObject.isSpirit())
|
||||
if (refArg && !dataObject.isSpirit())
|
||||
{
|
||||
_dataStore.createReference(commandObject, dataObject,DataStoreResources.model_contents);
|
||||
}
|
||||
|
|
@ -230,7 +230,7 @@ public class CommandGenerator
|
|||
commandObject.addNestedData(dataObject, false);
|
||||
}
|
||||
|
||||
if (!arg.isUpdated() || arg.isSpirit())
|
||||
if (!arg.isUpdated() || arg.isSpirit() || !refArg)
|
||||
{
|
||||
commandObject.addNestedData(arg, false);
|
||||
}
|
||||
|
|
@ -258,15 +258,14 @@ public class CommandGenerator
|
|||
*/
|
||||
public DataElement generateCommand(DataElement commandDescriptor, DataElement dataObject, boolean refArg)
|
||||
{
|
||||
_id++;
|
||||
|
||||
//refArg = false;
|
||||
DataElement commandObject = createCommand(commandDescriptor);
|
||||
if (commandObject != null)
|
||||
{
|
||||
commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName());
|
||||
|
||||
clearDeleted(dataObject);
|
||||
if ((refArg || dataObject.isUpdated()) && !dataObject.isSpirit())
|
||||
if (refArg && !dataObject.isSpirit())
|
||||
{
|
||||
_dataStore.createReference(commandObject, dataObject,DataStoreResources.model_arguments);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -382,6 +382,7 @@ public class ServerCommandHandler extends CommandHandler
|
|||
String property = dataObject.getName();
|
||||
String value = dataObject.getValue();
|
||||
_dataStore.setPreference(property, value);
|
||||
status.setAttribute(DE.A_NAME,DataStoreResources.model_done);
|
||||
}
|
||||
else if (commandName.equals(DataStoreSchema.C_QUERY_INSTALL))
|
||||
{
|
||||
|
|
|
|||
|
|
@ -19,26 +19,41 @@
|
|||
* David McKnight (IBM) - [331922] [dstore] enable DataElement recycling
|
||||
* David McKnight (IBM) - [371401] [dstore][multithread] avoid use of static variables - causes memory leak after disconnect
|
||||
* David McKnight (IBM) - [373507] [dstore][multithread] reduce heap memory on disconnect for server
|
||||
* David McKnight (IBM) - [385097] [dstore] DataStore spirit mechanism is not enabled
|
||||
* David McKnight (IBM) - [385097] [dstore] DataStore spirit mechanism is not enabled
|
||||
* David McKnight (IBM) - [390037] [dstore] Duplicated items in the System view
|
||||
* David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
|
||||
*******************************************************************************/
|
||||
|
||||
package org.eclipse.dstore.internal.core.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
|
||||
import org.eclipse.dstore.core.model.DataElement;
|
||||
import org.eclipse.dstore.core.model.DataStore;
|
||||
import org.eclipse.dstore.core.model.DataStoreResources;
|
||||
import org.eclipse.dstore.core.model.Handler;
|
||||
|
||||
|
||||
public class DataElementRemover extends Handler
|
||||
{
|
||||
private LinkedList _queue;
|
||||
private static int numDisconnected = 0;
|
||||
//private int numGCed = 0;
|
||||
{
|
||||
protected class QueueItem
|
||||
{
|
||||
public DataElement dataElement;
|
||||
public long timeStamp;
|
||||
|
||||
public QueueItem(DataElement element, long stamp)
|
||||
{
|
||||
dataElement = element;
|
||||
timeStamp = stamp;
|
||||
}
|
||||
|
||||
public boolean equals(QueueItem item){
|
||||
return item.dataElement == dataElement;
|
||||
}
|
||||
}
|
||||
|
||||
private ArrayList _queue;
|
||||
|
||||
// The following determine how DataElements are chosen to be removed once they
|
||||
// are in the queue for removal.
|
||||
|
|
@ -52,12 +67,19 @@ public class DataElementRemover extends Handler
|
|||
public static final String INTERVAL_TIME_PROPERTY_NAME = "SPIRIT_INTERVAL_TIME"; //$NON-NLS-1$
|
||||
public MemoryManager _memoryManager;
|
||||
|
||||
private int _lastLive = 0;
|
||||
private int _lastFree = 0;
|
||||
private long _lastMem = 0;
|
||||
|
||||
private boolean DEBUG = false; // extra tracing of hashmap when on
|
||||
private long _lastDumpTime = System.currentTimeMillis();
|
||||
|
||||
public DataElementRemover(DataStore dataStore)
|
||||
{
|
||||
super();
|
||||
_memoryManager = new MemoryManager(dataStore);
|
||||
_dataStore = dataStore;
|
||||
_queue = new LinkedList();
|
||||
_queue = new ArrayList();
|
||||
getTimes();
|
||||
setWaitTime(_intervalTime);
|
||||
}
|
||||
|
|
@ -84,41 +106,28 @@ public class DataElementRemover extends Handler
|
|||
}
|
||||
}
|
||||
|
||||
public static void addToRemovedCount()
|
||||
{
|
||||
// not using this anymore - better to get this from DataStore
|
||||
}
|
||||
|
||||
public static void addToCreatedCount()
|
||||
{
|
||||
// not using this anymore - better to get this from DataStore
|
||||
}
|
||||
|
||||
public static void addToGCedCount()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
public synchronized void addToQueueForRemoval(DataElement element)
|
||||
{
|
||||
synchronized (_queue)
|
||||
{
|
||||
if(isMemoryThresholdExceeded()) {
|
||||
if(element.isSpirit()) {
|
||||
unmap(element);
|
||||
public synchronized void addToQueueForRemoval(DataElement element){
|
||||
if(isMemoryThresholdExceeded()) {
|
||||
// do immediate clearing of queue since we're low on memory
|
||||
clearQueue(true);
|
||||
return;
|
||||
}
|
||||
if (_dataStore.isDoSpirit() &&
|
||||
!element.isReference() &&
|
||||
!element.isSpirit() &&
|
||||
!element.isDescriptor() &&
|
||||
!element.isDeleted()){
|
||||
//_dataStore.memLog("queuing " + element);
|
||||
QueueItem item = new QueueItem(element, System.currentTimeMillis());
|
||||
if (!_queue.contains(item)){
|
||||
synchronized (_queue){
|
||||
_queue.add(item);
|
||||
}
|
||||
|
||||
// do immediate clearing of queue since we're low on memory
|
||||
clearQueue(true);
|
||||
return;
|
||||
}
|
||||
if (_dataStore.isDoSpirit() && _dataStore == element.getDataStore())
|
||||
{
|
||||
QueueItem item = new QueueItem(element, System.currentTimeMillis());
|
||||
_queue.add(item);
|
||||
notifyInput();
|
||||
}
|
||||
}
|
||||
notifyInput();
|
||||
}
|
||||
|
||||
private boolean isMemoryThresholdExceeded(){
|
||||
|
|
@ -130,93 +139,79 @@ public class DataElementRemover extends Handler
|
|||
clearQueue(false);
|
||||
}
|
||||
|
||||
public synchronized void clearQueue(boolean force)
|
||||
{
|
||||
synchronized (_queue)
|
||||
{
|
||||
_dataStore.memLog(" "); //$NON-NLS-1$
|
||||
int disconnected = 0;
|
||||
if (!_dataStore.isDoSpirit())
|
||||
{
|
||||
if (_queue.size() > 0)
|
||||
{
|
||||
_dataStore.memLog("Clearing queue of size " + _queue.size() + ". DSTORE_SPIRIT_ON not set or set to false."); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
_queue.clear();
|
||||
}
|
||||
_dataStore.memLog("Total heap size: " + Runtime.getRuntime().totalMemory()); //$NON-NLS-1$
|
||||
_dataStore.memLog("Live elements: " + _dataStore.getNumElements()); //$NON-NLS-1$
|
||||
_dataStore.memLog("Recycled elements: " + _dataStore.getNumRecycled()); //$NON-NLS-1$
|
||||
_dataStore.memLog("Elements disconnected so far: " + numDisconnected); //$NON-NLS-1$
|
||||
|
||||
|
||||
// no longer a helpful stat since we no longer use finalize
|
||||
// _dataStore.memLog("DataElements GCed so far: " + numGCed); //$NON-NLS-1$
|
||||
return;
|
||||
}
|
||||
_dataStore.memLog("Total heap size before disconnection: " + Runtime.getRuntime().totalMemory()); //$NON-NLS-1$
|
||||
|
||||
_dataStore.memLog("Size of queue: " + _queue.size()); //$NON-NLS-1$
|
||||
|
||||
ArrayList toRefresh = new ArrayList();
|
||||
while (_queue.size() > 0 && (force || System.currentTimeMillis() - ((QueueItem) _queue.getFirst()).timeStamp > _expiryTime))
|
||||
{
|
||||
DataElement toBeDisconnected = ((QueueItem) _queue.removeFirst()).dataElement;
|
||||
if (!toBeDisconnected.isSpirit())
|
||||
{
|
||||
toBeDisconnected.setSpirit(true);
|
||||
toBeDisconnected.setUpdated(false);
|
||||
DataElement parent = toBeDisconnected.getParent();
|
||||
if (!toRefresh.contains(parent))
|
||||
{
|
||||
//System.out.println("disconnect parent:"+parent.getName());
|
||||
toRefresh.add(toBeDisconnected.getParent());
|
||||
}
|
||||
//_dataStore.refresh(toBeDisconnected);
|
||||
disconnected++;
|
||||
numDisconnected++;
|
||||
}
|
||||
else
|
||||
{
|
||||
//_dataStore.memLog(toBeDisconnected.toString());
|
||||
}
|
||||
unmap(toBeDisconnected);
|
||||
}
|
||||
|
||||
_dataStore.refresh(toRefresh);
|
||||
|
||||
_dataStore.memLog("Disconnected " + disconnected + " DataElements."); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
_dataStore.memLog("Live elements: " + _dataStore.getNumElements()); //$NON-NLS-1$
|
||||
_dataStore.memLog("Recycled elements: " + _dataStore.getNumRecycled()); //$NON-NLS-1$
|
||||
_dataStore.memLog("Elements disconnected so far: " + numDisconnected); //$NON-NLS-1$
|
||||
|
||||
// no longer a helpful stat since we no longer use finalize
|
||||
// _dataStore.memLog("DataElements GCed so far: " + numGCed); //$NON-NLS-1$
|
||||
System.gc();
|
||||
}
|
||||
}
|
||||
|
||||
private void unmap(DataElement element)
|
||||
{
|
||||
HashMap map = _dataStore.getHashMap();
|
||||
synchronized (map){
|
||||
map.remove(element.getId());
|
||||
_dataStore.addToRecycled(element);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected class QueueItem
|
||||
{
|
||||
public DataElement dataElement;
|
||||
public long timeStamp;
|
||||
private void logMemory(){
|
||||
long mem = Runtime.getRuntime().totalMemory();
|
||||
int liveElements = _dataStore.getNumElements();
|
||||
int freeElements = _dataStore.getNumRecycled();
|
||||
|
||||
public QueueItem(DataElement element, long stamp)
|
||||
{
|
||||
dataElement = element;
|
||||
timeStamp = stamp;
|
||||
if (mem != _lastMem || liveElements != _lastLive || freeElements != _lastFree){
|
||||
_dataStore.memLog(" "); //$NON-NLS-1$
|
||||
_dataStore.memLog("Total heap size: " + mem); //$NON-NLS-1$
|
||||
_dataStore.memLog("Number of live DataStore elements: " + liveElements); //$NON-NLS-1$
|
||||
_dataStore.memLog("Number of free DataStore elements: " + freeElements); //$NON-NLS-1$
|
||||
|
||||
_lastMem = mem;
|
||||
_lastLive = liveElements;
|
||||
_lastFree = freeElements;
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void clearQueue(boolean force){
|
||||
if (!_dataStore.isDoSpirit()){ // spiriting disabled
|
||||
if (_queue.size() > 0) {
|
||||
_dataStore.memLog("Clearing queue of size " + _queue.size() + ". DSTORE_SPIRIT_ON not set or set to false."); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
synchronized (_queue){
|
||||
_queue.clear();
|
||||
}
|
||||
}
|
||||
logMemory();
|
||||
return;
|
||||
}
|
||||
else { // spiriting enabled
|
||||
if (_queue.size() > 0){
|
||||
int queueSize = _queue.size();
|
||||
|
||||
ArrayList toRefresh = new ArrayList();
|
||||
long currentTime = System.currentTimeMillis();
|
||||
for (int i = queueSize - 1; i >= 0; i--){
|
||||
QueueItem qitem = null;
|
||||
synchronized (_queue){
|
||||
qitem = (QueueItem)_queue.get(i);
|
||||
}
|
||||
long deltaTime = currentTime - qitem.timeStamp;
|
||||
if (force || (deltaTime > _expiryTime)){
|
||||
DataElement toBeDisconnected = qitem.dataElement;
|
||||
toBeDisconnected.setSpirit(true);
|
||||
toBeDisconnected.setUpdated(false);
|
||||
DataElement parent = toBeDisconnected.getParent();
|
||||
if (!toRefresh.contains(parent)){
|
||||
toRefresh.add(parent);
|
||||
}
|
||||
synchronized (_queue){ // if spirited, dequeue
|
||||
_queue.remove(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!toRefresh.isEmpty()){
|
||||
// refresh parents of spirited items
|
||||
_dataStore.refresh(toRefresh);
|
||||
System.gc();
|
||||
}
|
||||
// print dump of elements on interval
|
||||
if (DEBUG){
|
||||
if (currentTime - _lastDumpTime > 100000){
|
||||
_lastDumpTime = currentTime;
|
||||
printHashmap();
|
||||
}
|
||||
}
|
||||
|
||||
logMemory();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Runs the handler loop in a thread.
|
||||
*/
|
||||
|
|
@ -238,5 +233,38 @@ public class DataElementRemover extends Handler
|
|||
handle();
|
||||
}
|
||||
}
|
||||
|
||||
// just used for tracing
|
||||
private void printHashmap(){
|
||||
_dataStore.memLog(" "); //$NON-NLS-1$
|
||||
_dataStore.memLog("------------------------------Current Hashmap--------------------------------:"); //$NON-NLS-1$
|
||||
HashMap map = _dataStore.getHashMap();
|
||||
synchronized (map){
|
||||
DataElement[] elements = (DataElement[])map.values().toArray(new DataElement[map.size()]);
|
||||
for (int i = 0; i < elements.length; i++){
|
||||
DataElement element = elements[i];
|
||||
if (!element.isDescriptor()){
|
||||
String type = element.getType();
|
||||
if (type.equals(DataStoreResources.model_abstracted_by) ||
|
||||
type.equals(DataStoreResources.model_abstracts) ||
|
||||
type.equals("Environment Variable") ||
|
||||
type.equals("system.property")){
|
||||
// schema and environment stuff
|
||||
}
|
||||
else {
|
||||
if (type.equals(DataStoreResources.model_status)){
|
||||
String value = element.getValue();
|
||||
DataElement parent = element.getParent();
|
||||
_dataStore.memLog("Command: " + parent.getName() + " is " + value);
|
||||
}
|
||||
else {
|
||||
_dataStore.memLog(element.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_dataStore.memLog("-----------------------------------------------------------------------------:"); //$NON-NLS-1$
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@
|
|||
* David McKnight (IBM) - [367449] [dstore] allow custom encoding for data transport layer
|
||||
* David McKnight (IBM) - [378136][dstore] miner.finish is stuck
|
||||
* David McKnight (IBM) - [391966][dstore][performance] unnecessary call slows down large queries
|
||||
* David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
|
||||
*******************************************************************************/
|
||||
|
||||
package org.eclipse.dstore.internal.core.util;
|
||||
|
|
@ -881,6 +882,7 @@ public class XMLparser
|
|||
if (attributes.length == DE.A_SIZE)
|
||||
{
|
||||
String type = attributes[DE.A_TYPE];
|
||||
String id = attributes[DE.A_ID];
|
||||
if (type.equals(DataStoreResources.KEEPALIVE_TYPE))
|
||||
{
|
||||
_isKeepAlive= true;
|
||||
|
|
@ -893,8 +895,7 @@ public class XMLparser
|
|||
}
|
||||
|
||||
else if (type.equals(DataStoreResources.DOCUMENT_TYPE))
|
||||
{
|
||||
String id = attributes[DE.A_ID];
|
||||
{
|
||||
if (_dataStore.contains(id))
|
||||
{
|
||||
result = _dataStore.find(id);
|
||||
|
|
@ -905,7 +906,12 @@ public class XMLparser
|
|||
result = _dataStore.createObject(null, attributes);
|
||||
}
|
||||
}
|
||||
|
||||
else if (id.equals(DataStoreResources.model_transient)){ // generic transient
|
||||
result = _dataStore.createTransientObject(attributes);
|
||||
if (parent != null){
|
||||
parent.addNestedData(result, false);
|
||||
}
|
||||
}
|
||||
else if (_isFile || _isClass || _isSerialized || parent == null)
|
||||
{
|
||||
result = _dataStore.createTransientObject(attributes);
|
||||
|
|
@ -934,13 +940,6 @@ public class XMLparser
|
|||
}
|
||||
else
|
||||
{
|
||||
String id = attributes[DE.A_ID];
|
||||
if (id == null)
|
||||
{
|
||||
handlePanic(new Exception(fullTag));
|
||||
return null;
|
||||
}
|
||||
|
||||
if (_dataStore.contains(id))
|
||||
{
|
||||
result = _dataStore.find(id);
|
||||
|
|
@ -963,7 +962,7 @@ public class XMLparser
|
|||
if (isSpirit)
|
||||
{
|
||||
if (!_dataStore.isVirtual()) attributes[DE.A_REF_TYPE] = DataStoreResources.VALUE;
|
||||
result.setSpirit(_dataStore.isVirtual());
|
||||
result.setSpirit(_dataStore.isVirtual()); // if this is the server, then the element is unspirited
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue