From 849647cef650d5f2384b9dfb86da6045b82db8ad Mon Sep 17 00:00:00 2001 From: David McKnight Date: Fri, 14 Dec 2012 13:22:27 -0500 Subject: [PATCH] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1) --- .../dstore/core/model/DataElement.java | 28 +- .../eclipse/dstore/core/model/DataStore.java | 117 +++----- .../dstore/core/model/DataStoreSchema.java | 4 +- .../dstore/core/model/UpdateHandler.java | 162 ++++++----- .../dstore/core/util/CommandGenerator.java | 19 +- .../core/server/ServerCommandHandler.java | 1 + .../core/util/DataElementRemover.java | 268 ++++++++++-------- .../dstore/internal/core/util/XMLparser.java | 21 +- 8 files changed, 312 insertions(+), 308 deletions(-) diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java index 86bedd4278c..79bf83cdcc5 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java @@ -19,6 +19,7 @@ * David McKnight (IBM) - [389286] [dstore] element delete should not clear _attributes since elements get recycled * David McKnight (IBM) - [390037] [dstore] Duplicated items in the System view * David McKnight (IBM) - [391065] [dstore][regression] DataElement with "" type treated as deleted (when it's not) + * David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1) *******************************************************************************/ package org.eclipse.dstore.core.model; @@ -28,7 +29,6 @@ import java.util.List; import org.eclipse.dstore.core.util.StringCompare; import org.eclipse.dstore.extra.IDataElement; -import org.eclipse.dstore.internal.core.util.DataElementRemover; import org.eclipse.dstore.internal.extra.DataElementActionFilter; import org.eclipse.dstore.internal.extra.DesktopElement; import org.eclipse.dstore.internal.extra.PropertySource; @@ -76,7 +76,6 @@ public final class DataElement implements IDataElement { _dataStore = null; _parent = null; - DataElementRemover.addToCreatedCount(); } /** @@ -88,7 +87,6 @@ public final class DataElement implements IDataElement { _dataStore = dataStore; _parent = null; - DataElementRemover.addToCreatedCount(); } /* Apparently having this method causes the GC to delay @@ -304,6 +302,7 @@ public final class DataElement implements IDataElement public void reInitAsTransient(String attributes[]) { _attributes = attributes; + setAttribute(DE.A_ID, DataStoreResources.model_transient); _isReference = false; _isDescriptor = false; @@ -886,14 +885,19 @@ public final class DataElement implements IDataElement */ public void setSpirit(boolean flag) { - _isSpirit = flag; - String refType = getAttribute(DE.A_REF_TYPE); - if (refType != null){ - if (_isSpirit && !refType.equals(DataStoreResources.SPIRIT)) { - setAttribute(DE.A_REF_TYPE, DataStoreResources.SPIRIT); - } - else if (refType.equals(DataStoreResources.SPIRIT)){ // if it was a spirit, change it back - setAttribute(DE.A_REF_TYPE, DataStoreResources.VALUE); + if (flag && isDescriptor()){ + // descriptors should not be spirited + } + else { + _isSpirit = flag; + String refType = getAttribute(DE.A_REF_TYPE); + if (refType != null){ + if (_isSpirit && !refType.equals(DataStoreResources.SPIRIT)) { + setAttribute(DE.A_REF_TYPE, DataStoreResources.SPIRIT); + } + else if (refType.equals(DataStoreResources.SPIRIT)){ // if it was a spirit, change it back + setAttribute(DE.A_REF_TYPE, DataStoreResources.VALUE); + } } } } @@ -1454,6 +1458,8 @@ public final class DataElement implements IDataElement + getSource() + "\n\tDepth:\t" //$NON-NLS-1$ + _depth + + "\n\tisSpirit:\t" //$NON-NLS-1$ + + _isSpirit + "\n\tDataStore:\t" //$NON-NLS-1$ + _dataStore.getName() + "\n}\n"; //$NON-NLS-1$ diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java index 2bb972c5a38..7d7bcc20b8c 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java @@ -45,6 +45,7 @@ * David McKnight (IBM) - [385097] [dstore] DataStore spirit mechanism is not enabled * David McKnight (IBM) - [385793] [dstore] DataStore spirit mechanism and other memory improvements needed * David McKnight (IBM) - [390037] [dstore] Duplicated items in the System view + * David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1) *******************************************************************************/ package org.eclipse.dstore.core.model; @@ -199,7 +200,7 @@ public final class DataStore * * This needs to be updated for each major release. */ - private String _RSE_version = "3.3.1"; //$NON-NLS-1$ + private String _RSE_version = "3.4.2"; //$NON-NLS-1$ /** * Creates a new DataStore instance @@ -950,10 +951,6 @@ public final class DataStore reference.reInit(parent, realObject, relationType); parent.addNestedData(reference, false); - String sugId = reference.getId(); - synchronized (_hashMap){ - _hashMap.put(sugId, reference); - } refresh(parent); return reference; @@ -1005,11 +1002,6 @@ public final class DataStore parent.addNestedData(reference, false); - String sugId = reference.getId(); - synchronized (_hashMap){ - _hashMap.put(sugId, reference); - } - if (doRefresh) { refresh(parent); @@ -1078,22 +1070,12 @@ public final class DataStore toReference.reInit(parent, realObject, toRelation); parent.addNestedData(toReference, false); - - String toId = toReference.getId(); - synchronized (_hashMap){ - _hashMap.put(toId, toReference); - } // reference with "from" relationship DataElement fromReference = createElement(); fromReference.reInit(realObject, parent, fromRelation); realObject.addNestedData(fromReference, false); - - String fromId = fromReference.getId(); - synchronized (_hashMap){ - _hashMap.put(fromId, fromReference); - } refresh(parent); @@ -1128,12 +1110,6 @@ public final class DataStore } parent.addNestedData(toReference, false); - - String toId = toReference.getId(); - synchronized (_hashMap){ - _hashMap.put(toId, toReference); - } - // reference with "from" relationship DataElement fromReference = createElement(); @@ -1149,10 +1125,6 @@ public final class DataStore realObject.addNestedData(fromReference, false); - String fromId = fromReference.getId(); - synchronized (_hashMap){ - _hashMap.put(fromId, fromReference); - } refresh(parent); @@ -1207,7 +1179,7 @@ public final class DataStore public DataElement createTransientObject(String attributes[]) { - DataElement newObject = createElement(); + DataElement newObject = new DataElement(this); newObject.reInitAsTransient(attributes); return newObject; @@ -2537,7 +2509,7 @@ public final class DataStore */ public DataElement command(DataElement commandDescriptor, ArrayList arguments, DataElement dataObject) { - return command(commandDescriptor, arguments, dataObject, false); + return command(commandDescriptor, arguments, dataObject, true); } /** @@ -2553,7 +2525,8 @@ public final class DataStore { if (_commandHandler != null) { - return _commandHandler.command(commandDescriptor, arguments, dataObject, true, immediate); + // as per bug #396440, default is now to not use references + return _commandHandler.command(commandDescriptor, arguments, dataObject, false, immediate); } return null; } @@ -2584,7 +2557,8 @@ public final class DataStore { if (_commandHandler != null) { - return _commandHandler.command(commandDescriptor, arg, dataObject, true, immediate); + // as per bug #396440, default is now to not use references + return _commandHandler.command(commandDescriptor, arg, dataObject, false, immediate); } return null; } @@ -2598,6 +2572,7 @@ public final class DataStore */ public DataElement command(DataElement commandDescriptor, DataElement dataObject) { + // as per bug #396440, default is now to not use references return command(commandDescriptor, dataObject, false); } @@ -2888,31 +2863,19 @@ public final class DataStore List searchList = root.getNestedData(); - - if (searchList != null) - { - for (int i = 0; i < searchList.size(); i++) - { + if (searchList != null){ + for (int i = 0; i < searchList.size(); i++){ DataElement child = (DataElement) searchList.get(i); - if (child != null) - { - synchronized (child) - { - if (child.isDeleted() && !results.contains(child)) - { - results.add(child); - if (!child.isReference()) - { - if (depth > 0) - { - List sResults = findDeleted(child, depth - 1); - for (int j = 0; j < sResults.size(); j++) - { - results.add(sResults.get(j)); - } + if (child != null){ + if (child.isDeleted() && !results.contains(child)){ + results.add(child); + if (!child.isReference()){ + if (depth > 0){ + List sResults = findDeleted(child, depth - 1); + for (int j = 0; j < sResults.size(); j++){ + results.add(sResults.get(j)); } - - } + } } } } @@ -3983,16 +3946,12 @@ public final class DataStore int numRecycled = _recycled.size(); - if (numRecycled > 1) - { - synchronized (_recycled) - { - if (numRecycled > _MAX_FREE) - { + if (numRecycled > 0){ + synchronized (_recycled){ + if (numRecycled > _MAX_FREE){ int numRemoved = numRecycled - _MAX_FREE; - for (int i = 1; i <= numRemoved; i++) - { - _recycled.remove(numRemoved - i); + for (int i = numRemoved - 1; i >=0; i--){ + _recycled.remove(i); } } newObject = (DataElement) _recycled.remove((_recycled.size() - 1)); @@ -4076,16 +4035,16 @@ public final class DataStore private void disconnectObjectHelper(DataElement toDisconnect, int depth) { - if (depth > 0) - { + if (depth > 0){ depth--; _deRemover.addToQueueForRemoval(toDisconnect); - for (int i = 0; i < toDisconnect.getNestedSize(); i++) - { - DataElement subDisconnect = toDisconnect.get(i); - if (subDisconnect != null && subDisconnect.getDataStore() == this && !subDisconnect.isSpirit()) - { - disconnectObjectHelper(subDisconnect, depth); + List nestedData = toDisconnect.getNestedData(); + if (nestedData != null){ + for (int i = 0; i < nestedData.size(); i++){ + DataElement subDisconnect = (DataElement)nestedData.get(i); + if (subDisconnect != null && !subDisconnect.isSpirit() && !subDisconnect.isDescriptor() && !subDisconnect.isReference()){ + disconnectObjectHelper(subDisconnect, depth); + } } } } @@ -4105,18 +4064,8 @@ public final class DataStore else { return generateId(); - /* - String newId = String.valueOf(_random.nextInt()); - while (_hashMap.containsKey(newId)) - { - newId = String.valueOf(_random.nextInt()); - } - - return newId; - */ } - } private String generateId(DataElement parent, String type, String name) diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStoreSchema.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStoreSchema.java index 6ebaa25e8f7..ed1f9213eca 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStoreSchema.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStoreSchema.java @@ -13,6 +13,7 @@ * Contributors: * David McKnight (IBM) - [226561] [apidoc] Add API markup to RSE Javadocs where extend / implement is allowed * David McKnight (IBM) - [385097] [dstore] DataStore spirit mechanism is not enabled + * David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1) ********************************************************************************/ package org.eclipse.dstore.core.model; @@ -191,9 +192,6 @@ public class DataStoreSchema // miner-specific descriptors are defined in the miners when they extend the schema // these first elements are the most fundamental -// DataElement uiCmdD = _dataStore.createObject(schemaRoot, DE.T_UI_COMMAND_DESCRIPTOR, DE.T_UI_COMMAND_DESCRIPTOR); - _dataStore.createObject(schemaRoot, DE.T_UI_COMMAND_DESCRIPTOR, DE.T_UI_COMMAND_DESCRIPTOR); - _commandDescriptor = _dataStore.createCommandDescriptor(schemaRoot, DE.T_COMMAND_DESCRIPTOR); _objectDescriptor = _dataStore.createObjectDescriptor(schemaRoot, DE.T_OBJECT_DESCRIPTOR); _relationDescriptor = _dataStore.createRelationDescriptor(schemaRoot, DE.T_RELATION_DESCRIPTOR); diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java index 922f5e8c56a..2e171f48a74 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java @@ -17,15 +17,16 @@ * David McKnight (IBM) - [385793] [dstore] DataStore spirit mechanism and other memory improvements needed * David McKnight (IBM) - [389286] [dstore] element delete should not clear _attributes since elements get recycled * David McKnight (IBM) - [390037] [dstore] Duplicated items in the System view + * David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1) *******************************************************************************/ package org.eclipse.dstore.core.model; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import org.eclipse.dstore.core.java.IRemoteClassInstance; -import org.eclipse.dstore.internal.core.util.DataElementRemover; /** *

@@ -75,49 +76,83 @@ public abstract class UpdateHandler extends Handler clean(object, 2); } + + private void unmap(DataElement element) + { + if (element.isDeleted() || element.isSpirit()){ + HashMap map = _dataStore.getHashMap(); + synchronized (map){ + String id = element.getId(); + // _dataStore.memLog("unmapping " + id); + map.remove(id); + _dataStore.addToRecycled(element); + } + } + } + protected void clean(DataElement object, int depth) { - if ((depth > 0) && (object != null) && object.getNestedSize() > 0) - { - List deletedList = _dataStore.findDeleted(object); + boolean isServer = !_dataStore.isVirtual(); + if ((depth > 0) && object != null){ + if (object.isSpirit() || object.isDeleted()){ + DataElement parent = object.getParent(); + cleanChildren(object); + unmap(object); + if (object.isSpirit() && isServer){ + // officially delete this now + object.delete(); + } + if (isServer){ + object.clear(); + } + if (parent != null){ + synchronized (parent){ + parent.removeNestedData(object); + } + } + } + else if (object.getNestedSize() > 0){ + cleanChildren(object); + } + } + // remove objects under temproot + DataElement tmpRoot = _dataStore.getTempRoot(); + if (tmpRoot != null){ + tmpRoot.removeNestedData(); + } + } - for (int i = 0; i < deletedList.size(); i++) - { + + /* + protected void clean(DataElement object, int depth) + { + boolean isServer = !_dataStore.isVirtual(); + if ((depth > 0) && object != null){ + List deletedList = _dataStore.findDeleted(object); + for (int i = 0; i < deletedList.size(); i++){ DataElement child = (DataElement) deletedList.get(i); - if (child != null && child.isDeleted()) - { + if (child != null && child.isDeleted()){ DataElement parent = child.getParent(); - DataElementRemover.addToRemovedCount(); cleanChildren(child); // clean the children - - - if (child.isSpirit()) - { + unmap(child); + if (child.isSpirit() && isServer){ // officially delete this now - // will only happen on server since, on client, - // the above call to isDeleted() returns false for spirited child.delete(); } - child.clear(); - if (parent != null) - { - synchronized (parent) - { + if (parent != null){ + synchronized (parent){ parent.removeNestedData(child); } } - // _dataStore.addToRecycled(child); } } deletedList.clear(); } - // delete objects under temproot - _dataStore.getTempRoot().removeNestedData(); - } +*/ /** * Recursively clean children for deletion @@ -125,25 +160,26 @@ public abstract class UpdateHandler extends Handler */ protected void cleanChildren(DataElement parent) { + boolean isServer = !_dataStore.isVirtual(); List nestedData = parent.getNestedData(); - if (nestedData != null) - { + if (nestedData != null){ synchronized (nestedData){ for (int i = nestedData.size() - 1; i >= 0; i--){ DataElement child = (DataElement)nestedData.get(i); - cleanChildren(child); - - if (child.isSpirit()) - { - // officially delete this now - child.delete(); + if (child.isSpirit() || child.isDeleted()){ + cleanChildren(child); + unmap(child); + + if (isServer){ + // officially delete this now + child.delete(); + child.clear(); + } + nestedData.remove(child); } - - child.clear(); - parent.removeNestedData(child); } } - } + } } /** @@ -172,51 +208,39 @@ public abstract class UpdateHandler extends Handler * @param object an object to get updated * @param immediate true indicates that this object should be first in the queue */ - public void update(DataElement object, boolean immediate) - { - synchronized (_dataObjects) - { - if (object != null){ - String type = object.getType(); - boolean isStatus = DataStoreResources.model_status.equals(type); - if (immediate) - { - _dataObjects.add(0, object); - // none of this immediate stuff - just put it at the beginning - //handle(); + public void update(DataElement object, boolean immediate){ + if (object != null){ + String type = object.getType(); + boolean statusDone = false; + boolean isStatus = DataStoreResources.model_status.equals(type); + if (isStatus){ + statusDone = DataStoreResources.model_done.equals(object.getName()) || DataStoreResources.model_done.equals(object.getValue()); } - else - { - if (!_dataObjects.contains(object)) - { - _dataObjects.add(object); + synchronized (_dataObjects){ + if (immediate){ + _dataObjects.add(0, object); } - else - { - - if (_dataStore != null && object != null && !object.isDeleted()) - { - if (isStatus) - { - if (object.getName().equals(DataStoreResources.model_done)) - { - //DKM + else { + if (!_dataObjects.contains(object)){ + _dataObjects.add(object); + } + else { + if (_dataStore != null && object != null && !object.isDeleted()){ + if (isStatus && statusDone){ // move to the back of the queue // this is done so that if status that was already queued changed to done in between // requests, and had not yet been transferred over comm layer, the completed status // object does not come back to client (as "done") before the results of a query _dataObjects.remove(object); - _dataObjects.add(object); - + _dataObjects.add(object); } } } } } - if (_dataStore != null && !_dataStore.isVirtual() && isStatus){ - _dataStore.disconnectObjects(object); // spirit the status - } - + if (_dataStore != null && !_dataStore.isVirtual() && isStatus && statusDone){ + _dataStore.disconnectObject(object.getParent()); // spirit the command and its children + // _dataStore.disconnectObject(object); } } notifyInput(); diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/CommandGenerator.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/CommandGenerator.java index 7beeae5fcfb..ca3a83a9471 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/CommandGenerator.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/CommandGenerator.java @@ -15,6 +15,7 @@ * David McKnight (IBM) - [225507][api][breaking] RSE dstore API leaks non-API types * David McKnight (IBM) - [226561] [apidoc] Add API markup to RSE Javadocs where extend / implement is allowed * David McKnight (IBM) - [390037] [dstore] Duplicated items in the System view + * David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1) *******************************************************************************/ package org.eclipse.dstore.core.util; @@ -43,7 +44,6 @@ public class CommandGenerator private DataStore _dataStore = null; private DataElement _log = null; - static private int _id = 0; /** * Constructor @@ -143,7 +143,7 @@ public class CommandGenerator */ public DataElement generateCommand(DataElement commandDescriptor, ArrayList arguments, DataElement dataObject, boolean refArg) { - + //refArg = false; DataElement commandObject = createCommand(commandDescriptor); if (commandObject != null) { @@ -151,7 +151,7 @@ public class CommandGenerator commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName()); - if (dataObject.isUpdated() && !dataObject.isSpirit()) + if (refArg && !dataObject.isSpirit()) { _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_contents); } @@ -175,7 +175,7 @@ public class CommandGenerator DataElement arg = (DataElement) arguments.get(i); if (arg != null) { - if (!arg.isUpdated() || arg.isSpirit()) + if (!arg.isUpdated() || arg.isSpirit() || !refArg) { commandObject.addNestedData(arg, false); } @@ -207,14 +207,14 @@ public class CommandGenerator */ public DataElement generateCommand(DataElement commandDescriptor, DataElement arg, DataElement dataObject, boolean refArg) { - _id++; + //refArg = false; DataElement commandObject = createCommand(commandDescriptor); if (commandObject != null) { commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName()); clearDeleted(dataObject); - if ((refArg || dataObject.isUpdated()) && !dataObject.isSpirit()) + if (refArg && !dataObject.isSpirit()) { _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_contents); } @@ -230,7 +230,7 @@ public class CommandGenerator commandObject.addNestedData(dataObject, false); } - if (!arg.isUpdated() || arg.isSpirit()) + if (!arg.isUpdated() || arg.isSpirit() || !refArg) { commandObject.addNestedData(arg, false); } @@ -258,15 +258,14 @@ public class CommandGenerator */ public DataElement generateCommand(DataElement commandDescriptor, DataElement dataObject, boolean refArg) { - _id++; - + //refArg = false; DataElement commandObject = createCommand(commandDescriptor); if (commandObject != null) { commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName()); clearDeleted(dataObject); - if ((refArg || dataObject.isUpdated()) && !dataObject.isSpirit()) + if (refArg && !dataObject.isSpirit()) { _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_arguments); } diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/server/ServerCommandHandler.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/server/ServerCommandHandler.java index c72d048abec..5a4dc99bed2 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/server/ServerCommandHandler.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/server/ServerCommandHandler.java @@ -382,6 +382,7 @@ public class ServerCommandHandler extends CommandHandler String property = dataObject.getName(); String value = dataObject.getValue(); _dataStore.setPreference(property, value); + status.setAttribute(DE.A_NAME,DataStoreResources.model_done); } else if (commandName.equals(DataStoreSchema.C_QUERY_INSTALL)) { diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java index 86c0bee6bd1..7cb07107cf6 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java @@ -19,26 +19,41 @@ * David McKnight (IBM) - [331922] [dstore] enable DataElement recycling * David McKnight (IBM) - [371401] [dstore][multithread] avoid use of static variables - causes memory leak after disconnect * David McKnight (IBM) - [373507] [dstore][multithread] reduce heap memory on disconnect for server - * David McKnight (IBM) - [385097] [dstore] DataStore spirit mechanism is not enabled + * David McKnight (IBM) - [385097] [dstore] DataStore spirit mechanism is not enabled * David McKnight (IBM) - [390037] [dstore] Duplicated items in the System view + * David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1) *******************************************************************************/ package org.eclipse.dstore.internal.core.util; import java.util.ArrayList; import java.util.HashMap; -import java.util.LinkedList; import org.eclipse.dstore.core.model.DataElement; import org.eclipse.dstore.core.model.DataStore; +import org.eclipse.dstore.core.model.DataStoreResources; import org.eclipse.dstore.core.model.Handler; public class DataElementRemover extends Handler -{ - private LinkedList _queue; - private static int numDisconnected = 0; - //private int numGCed = 0; +{ + protected class QueueItem + { + public DataElement dataElement; + public long timeStamp; + + public QueueItem(DataElement element, long stamp) + { + dataElement = element; + timeStamp = stamp; + } + + public boolean equals(QueueItem item){ + return item.dataElement == dataElement; + } + } + + private ArrayList _queue; // The following determine how DataElements are chosen to be removed once they // are in the queue for removal. @@ -52,12 +67,19 @@ public class DataElementRemover extends Handler public static final String INTERVAL_TIME_PROPERTY_NAME = "SPIRIT_INTERVAL_TIME"; //$NON-NLS-1$ public MemoryManager _memoryManager; + private int _lastLive = 0; + private int _lastFree = 0; + private long _lastMem = 0; + + private boolean DEBUG = false; // extra tracing of hashmap when on + private long _lastDumpTime = System.currentTimeMillis(); + public DataElementRemover(DataStore dataStore) { super(); _memoryManager = new MemoryManager(dataStore); _dataStore = dataStore; - _queue = new LinkedList(); + _queue = new ArrayList(); getTimes(); setWaitTime(_intervalTime); } @@ -84,41 +106,28 @@ public class DataElementRemover extends Handler } } - public static void addToRemovedCount() - { - // not using this anymore - better to get this from DataStore - } - - public static void addToCreatedCount() - { - // not using this anymore - better to get this from DataStore - } - - public static void addToGCedCount() - { - } - public synchronized void addToQueueForRemoval(DataElement element) - { - synchronized (_queue) - { - if(isMemoryThresholdExceeded()) { - if(element.isSpirit()) { - unmap(element); + public synchronized void addToQueueForRemoval(DataElement element){ + if(isMemoryThresholdExceeded()) { + // do immediate clearing of queue since we're low on memory + clearQueue(true); + return; + } + if (_dataStore.isDoSpirit() && + !element.isReference() && + !element.isSpirit() && + !element.isDescriptor() && + !element.isDeleted()){ + //_dataStore.memLog("queuing " + element); + QueueItem item = new QueueItem(element, System.currentTimeMillis()); + if (!_queue.contains(item)){ + synchronized (_queue){ + _queue.add(item); } - - // do immediate clearing of queue since we're low on memory - clearQueue(true); - return; - } - if (_dataStore.isDoSpirit() && _dataStore == element.getDataStore()) - { - QueueItem item = new QueueItem(element, System.currentTimeMillis()); - _queue.add(item); + notifyInput(); } } - notifyInput(); } private boolean isMemoryThresholdExceeded(){ @@ -130,93 +139,79 @@ public class DataElementRemover extends Handler clearQueue(false); } - public synchronized void clearQueue(boolean force) - { - synchronized (_queue) - { - _dataStore.memLog(" "); //$NON-NLS-1$ - int disconnected = 0; - if (!_dataStore.isDoSpirit()) - { - if (_queue.size() > 0) - { - _dataStore.memLog("Clearing queue of size " + _queue.size() + ". DSTORE_SPIRIT_ON not set or set to false."); //$NON-NLS-1$ //$NON-NLS-2$ - _queue.clear(); - } - _dataStore.memLog("Total heap size: " + Runtime.getRuntime().totalMemory()); //$NON-NLS-1$ - _dataStore.memLog("Live elements: " + _dataStore.getNumElements()); //$NON-NLS-1$ - _dataStore.memLog("Recycled elements: " + _dataStore.getNumRecycled()); //$NON-NLS-1$ - _dataStore.memLog("Elements disconnected so far: " + numDisconnected); //$NON-NLS-1$ - - - // no longer a helpful stat since we no longer use finalize - // _dataStore.memLog("DataElements GCed so far: " + numGCed); //$NON-NLS-1$ - return; - } - _dataStore.memLog("Total heap size before disconnection: " + Runtime.getRuntime().totalMemory()); //$NON-NLS-1$ - - _dataStore.memLog("Size of queue: " + _queue.size()); //$NON-NLS-1$ - - ArrayList toRefresh = new ArrayList(); - while (_queue.size() > 0 && (force || System.currentTimeMillis() - ((QueueItem) _queue.getFirst()).timeStamp > _expiryTime)) - { - DataElement toBeDisconnected = ((QueueItem) _queue.removeFirst()).dataElement; - if (!toBeDisconnected.isSpirit()) - { - toBeDisconnected.setSpirit(true); - toBeDisconnected.setUpdated(false); - DataElement parent = toBeDisconnected.getParent(); - if (!toRefresh.contains(parent)) - { - //System.out.println("disconnect parent:"+parent.getName()); - toRefresh.add(toBeDisconnected.getParent()); - } - //_dataStore.refresh(toBeDisconnected); - disconnected++; - numDisconnected++; - } - else - { - //_dataStore.memLog(toBeDisconnected.toString()); - } - unmap(toBeDisconnected); - } - - _dataStore.refresh(toRefresh); - - _dataStore.memLog("Disconnected " + disconnected + " DataElements."); //$NON-NLS-1$ //$NON-NLS-2$ - _dataStore.memLog("Live elements: " + _dataStore.getNumElements()); //$NON-NLS-1$ - _dataStore.memLog("Recycled elements: " + _dataStore.getNumRecycled()); //$NON-NLS-1$ - _dataStore.memLog("Elements disconnected so far: " + numDisconnected); //$NON-NLS-1$ - - // no longer a helpful stat since we no longer use finalize - // _dataStore.memLog("DataElements GCed so far: " + numGCed); //$NON-NLS-1$ - System.gc(); - } - } - - private void unmap(DataElement element) - { - HashMap map = _dataStore.getHashMap(); - synchronized (map){ - map.remove(element.getId()); - _dataStore.addToRecycled(element); - } - } - - - protected class QueueItem - { - public DataElement dataElement; - public long timeStamp; + private void logMemory(){ + long mem = Runtime.getRuntime().totalMemory(); + int liveElements = _dataStore.getNumElements(); + int freeElements = _dataStore.getNumRecycled(); - public QueueItem(DataElement element, long stamp) - { - dataElement = element; - timeStamp = stamp; + if (mem != _lastMem || liveElements != _lastLive || freeElements != _lastFree){ + _dataStore.memLog(" "); //$NON-NLS-1$ + _dataStore.memLog("Total heap size: " + mem); //$NON-NLS-1$ + _dataStore.memLog("Number of live DataStore elements: " + liveElements); //$NON-NLS-1$ + _dataStore.memLog("Number of free DataStore elements: " + freeElements); //$NON-NLS-1$ + + _lastMem = mem; + _lastLive = liveElements; + _lastFree = freeElements; } } + public synchronized void clearQueue(boolean force){ + if (!_dataStore.isDoSpirit()){ // spiriting disabled + if (_queue.size() > 0) { + _dataStore.memLog("Clearing queue of size " + _queue.size() + ". DSTORE_SPIRIT_ON not set or set to false."); //$NON-NLS-1$ //$NON-NLS-2$ + synchronized (_queue){ + _queue.clear(); + } + } + logMemory(); + return; + } + else { // spiriting enabled + if (_queue.size() > 0){ + int queueSize = _queue.size(); + + ArrayList toRefresh = new ArrayList(); + long currentTime = System.currentTimeMillis(); + for (int i = queueSize - 1; i >= 0; i--){ + QueueItem qitem = null; + synchronized (_queue){ + qitem = (QueueItem)_queue.get(i); + } + long deltaTime = currentTime - qitem.timeStamp; + if (force || (deltaTime > _expiryTime)){ + DataElement toBeDisconnected = qitem.dataElement; + toBeDisconnected.setSpirit(true); + toBeDisconnected.setUpdated(false); + DataElement parent = toBeDisconnected.getParent(); + if (!toRefresh.contains(parent)){ + toRefresh.add(parent); + } + synchronized (_queue){ // if spirited, dequeue + _queue.remove(i); + } + } + } + if (!toRefresh.isEmpty()){ + // refresh parents of spirited items + _dataStore.refresh(toRefresh); + System.gc(); + } + // print dump of elements on interval + if (DEBUG){ + if (currentTime - _lastDumpTime > 100000){ + _lastDumpTime = currentTime; + printHashmap(); + } + } + + logMemory(); + } + } + } + + + /** * Runs the handler loop in a thread. */ @@ -238,5 +233,38 @@ public class DataElementRemover extends Handler handle(); } } + + // just used for tracing + private void printHashmap(){ + _dataStore.memLog(" "); //$NON-NLS-1$ + _dataStore.memLog("------------------------------Current Hashmap--------------------------------:"); //$NON-NLS-1$ + HashMap map = _dataStore.getHashMap(); + synchronized (map){ + DataElement[] elements = (DataElement[])map.values().toArray(new DataElement[map.size()]); + for (int i = 0; i < elements.length; i++){ + DataElement element = elements[i]; + if (!element.isDescriptor()){ + String type = element.getType(); + if (type.equals(DataStoreResources.model_abstracted_by) || + type.equals(DataStoreResources.model_abstracts) || + type.equals("Environment Variable") || + type.equals("system.property")){ + // schema and environment stuff + } + else { + if (type.equals(DataStoreResources.model_status)){ + String value = element.getValue(); + DataElement parent = element.getParent(); + _dataStore.memLog("Command: " + parent.getName() + " is " + value); + } + else { + _dataStore.memLog(element.toString()); + } + } + } + } + } + _dataStore.memLog("-----------------------------------------------------------------------------:"); //$NON-NLS-1$ + } } diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/XMLparser.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/XMLparser.java index 15c9db7ba34..4aa6ca807ae 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/XMLparser.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/XMLparser.java @@ -28,6 +28,7 @@ * David McKnight (IBM) - [367449] [dstore] allow custom encoding for data transport layer * David McKnight (IBM) - [378136][dstore] miner.finish is stuck * David McKnight (IBM) - [391966][dstore][performance] unnecessary call slows down large queries + * David McKnight (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1) *******************************************************************************/ package org.eclipse.dstore.internal.core.util; @@ -881,6 +882,7 @@ public class XMLparser if (attributes.length == DE.A_SIZE) { String type = attributes[DE.A_TYPE]; + String id = attributes[DE.A_ID]; if (type.equals(DataStoreResources.KEEPALIVE_TYPE)) { _isKeepAlive= true; @@ -893,8 +895,7 @@ public class XMLparser } else if (type.equals(DataStoreResources.DOCUMENT_TYPE)) - { - String id = attributes[DE.A_ID]; + { if (_dataStore.contains(id)) { result = _dataStore.find(id); @@ -905,7 +906,12 @@ public class XMLparser result = _dataStore.createObject(null, attributes); } } - + else if (id.equals(DataStoreResources.model_transient)){ // generic transient + result = _dataStore.createTransientObject(attributes); + if (parent != null){ + parent.addNestedData(result, false); + } + } else if (_isFile || _isClass || _isSerialized || parent == null) { result = _dataStore.createTransientObject(attributes); @@ -934,13 +940,6 @@ public class XMLparser } else { - String id = attributes[DE.A_ID]; - if (id == null) - { - handlePanic(new Exception(fullTag)); - return null; - } - if (_dataStore.contains(id)) { result = _dataStore.find(id); @@ -963,7 +962,7 @@ public class XMLparser if (isSpirit) { if (!_dataStore.isVirtual()) attributes[DE.A_REF_TYPE] = DataStoreResources.VALUE; - result.setSpirit(_dataStore.isVirtual()); + result.setSpirit(_dataStore.isVirtual()); // if this is the server, then the element is unspirited } else {