diff --git a/rse/plugins/org.eclipse.dstore.core/export.jardesc b/rse/plugins/org.eclipse.dstore.core/export.jardesc new file mode 100644 index 00000000000..6eebacfac36 --- /dev/null +++ b/rse/plugins/org.eclipse.dstore.core/export.jardesc @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DE.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DE.java index 53e341c1b4f..31238091a33 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DE.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DE.java @@ -105,9 +105,14 @@ public class DE public static final String P_VALUE = "value"; /* - * The is reference? property identifier of a DataElement. + * The is reference? property identifier of a DataElement. Deprecated. Use P_REF_TYPE. */ public static final String P_ISREF = "isRef"; + + /* + * The is reference? property identifier of a DataElement. + */ + public static final String P_REF_TYPE = "refType"; /* * The visibility property identifier of a DataElement. @@ -207,9 +212,14 @@ public class DE public static final int A_SOURCE_LOCATION = 5; /* - * IsRef attribute index. + * IsRef attribute index. Deprecated. Use A_REF_TYPE. */ public static final int A_ISREF = 6; + + /* + * RefType attribute index. + */ + public static final int A_REF_TYPE = 6; /* * Visibility attribute index. diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java index b9255fe16f8..9c44fcfddb7 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java @@ -19,6 +19,7 @@ package org.eclipse.dstore.core.model; import java.util.ArrayList; import java.util.List; +import org.eclipse.dstore.core.util.DataElementRemover; import org.eclipse.dstore.core.util.StringCompare; import org.eclipse.dstore.extra.internal.extra.DataElementActionFilter; import org.eclipse.dstore.extra.internal.extra.DesktopElement; @@ -44,6 +45,7 @@ public final class DataElement implements IDataElement private boolean _isExpanded = false; private boolean _isUpdated = false; private boolean _isPendingTransfer = false; + private boolean _isSpirit = false; private int _depth = 1; @@ -65,6 +67,7 @@ public final class DataElement implements IDataElement { _dataStore = null; _parent = null; + DataElementRemover.addToCreatedCount(); } /** @@ -76,7 +79,12 @@ public final class DataElement implements IDataElement { _dataStore = dataStore; _parent = null; - + DataElementRemover.addToCreatedCount(); + } + + protected void finalize() + { + DataElementRemover.addToGCedCount(); } /** @@ -324,16 +332,16 @@ public final class DataElement implements IDataElement String valueAttribute = getAttribute(DE.A_VALUE); - if (_depth == -1) + if (valueAttribute != null && valueAttribute.equals(DataStoreResources.DELETED)) { return true; } - else if (valueAttribute != null && valueAttribute.equals(DataStoreResources.DELETED)) + + if (_isSpirit && !_dataStore.isVirtual()) { - _depth = -1; - return true; + return true; } - + return false; } @@ -726,6 +734,18 @@ public final class DataElement implements IDataElement return _isPendingTransfer; } + /** + * Indicates whether this element is a 'spirit' element. + * In a client datastore, this means that the element's counterpart on the + * server is either also a spirit (and will be deleted soon) or has already + * been deleted. In a server datastore, this means that the element is + * to be deleted at the next opportunity in order to free memory. + */ + public boolean isSpirit() + { + return _isSpirit; + } + /** * Sets an attribute of the element. * @@ -837,6 +857,18 @@ public final class DataElement implements IDataElement { _isPendingTransfer = flag; } + + /** + * Sets indication of whether this element is a 'spirit' element. + * In a client datastore, this means that the element's counterpart on the + * server is either also a spirit (and will be deleted soon) or has already + * been deleted. In a server datastore, this means that the element is + * to be deleted at the next opportunity in order to free memory. + */ + public void setSpirit(boolean flag) + { + _isSpirit = flag; + } /** * Sets the parent for this element. @@ -1534,10 +1566,24 @@ public final class DataElement implements IDataElement - String isRef = getAttribute(DE.A_ISREF); - if (isRef != null && isRef.equals(DataStoreResources.TRUE)) + String refType = getAttribute(DE.A_REF_TYPE); + if (refType != null) { - _isReference = true; + if (refType.equals(DataStoreResources.TRUE) || refType.equals(DataStoreResources.REFERENCE)) + { + _isReference = true; + _isSpirit = false; + } + else if (refType.equals(DataStoreResources.FALSE) || refType.equals(DataStoreResources.VALUE)) + { + _isReference = false; + _isSpirit = false; + } + else if (refType.equals(DataStoreResources.SPIRIT)) + { + _isReference = false; + _isSpirit = true; + } } String type = getAttribute(DE.A_TYPE); @@ -1608,7 +1654,7 @@ public final class DataElement implements IDataElement _nestedData = null; } } - + public synchronized void notifyUpdate() { notify(); diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java index cd07dd7328e..023d09fc8e4 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java @@ -28,6 +28,7 @@ import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.RandomAccessFile; import java.util.ArrayList; +import java.util.Date; import java.util.Enumeration; import java.util.HashMap; import java.util.List; @@ -42,6 +43,7 @@ import org.eclipse.dstore.core.java.ClassByteStreamHandlerRegistry; import org.eclipse.dstore.core.java.IClassByteStreamHandler; import org.eclipse.dstore.core.java.IRemoteClassInstance; import org.eclipse.dstore.core.java.RemoteClassLoader; +import org.eclipse.dstore.core.util.DataElementRemover; import org.eclipse.dstore.core.util.ExternalLoader; import org.eclipse.dstore.core.util.StringCompare; import org.eclipse.dstore.core.util.XMLgenerator; @@ -119,6 +121,12 @@ public final class DataStore private File _traceFileHandle; private RandomAccessFile _traceFile; private boolean _tracingOn; + + private boolean _spiritModeOn = false; + private boolean _spiritCommandReceived = false; + private File _memLoggingFileHandle; + private RandomAccessFile _memLogFile; + private boolean _memLoggingOn; private ArrayList _waitingStatuses = null; @@ -128,6 +136,9 @@ public final class DataStore private File _cacheJar; public static final String REMOTE_CLASS_CACHE_JARFILE_NAME = "rmt_classloader_cache"; public static final String JARFILE_EXTENSION = ".jar"; + private DataElementRemover _deRemover; + public static final int SPIRIT_ON_INITIAL_SIZE = 1000; + private String referenceTag = null; private int _serverVersion; private int _serverMinor; @@ -147,8 +158,8 @@ public final class DataStore _domainNotifier = null; _isConnected = false; _logTimes = false; - _initialSize = 100000; - + setSpiritModeOnState(); + _initialSize = _spiritModeOn && !isVirtual() ? SPIRIT_ON_INITIAL_SIZE : 100000; initialize(); } @@ -166,8 +177,8 @@ public final class DataStore _domainNotifier = null; _isConnected = false; _logTimes = false; - _initialSize = initialSize; - + setSpiritModeOnState(); + _initialSize = _spiritModeOn && !isVirtual() ? SPIRIT_ON_INITIAL_SIZE : initialSize; initialize(); } @@ -187,8 +198,8 @@ public final class DataStore _domainNotifier = domainNotifier; _isConnected = true; _logTimes = false; - _initialSize = 10000; - + setSpiritModeOnState(); + _initialSize = _spiritModeOn && !isVirtual() ? SPIRIT_ON_INITIAL_SIZE : 10000; initialize(); createRoot(); } @@ -210,12 +221,21 @@ public final class DataStore _domainNotifier = domainNotifier; _isConnected = true; _logTimes = false; - _initialSize = initialSize; - + setSpiritModeOnState(); + _initialSize = _spiritModeOn && !isVirtual() ? SPIRIT_ON_INITIAL_SIZE : initialSize; initialize(); createRoot(); } + protected void setSpiritModeOnState() + { + if (isVirtual()) _spiritModeOn = true; + else + { + String doSpirit = System.getProperty("DSTORE_SPIRIT_ON"); + _spiritModeOn = (doSpirit != null && doSpirit.equals("true")); + } + } public void setServerVersion(int version) { @@ -1625,6 +1645,28 @@ public final class DataStore // refresh(from); } } + + /** + * Disconnect all the elements contained in from + * + * @param from the element from which to disconnect objects + */ + public void disconnectObjects(DataElement from) + { + if (from != null) + { + for (int i = from.getNestedSize() - 1; i >= 0; i--) + { + DataElement disconnectee = from.get(i); + if (disconnectee != null) + { + disconnectObjectHelper(disconnectee, 5); + } + } + + // refresh(from); + } + } /** * Deletes an element from another element @@ -1641,6 +1683,21 @@ public final class DataStore // refresh(from); } } + + /** + * Disconnects an element and makes it a "spirit" + * + * @param toDisconnect the element to disconnect + */ + public void disconnectObject(DataElement toDisconnect) + { + if (toDisconnect != null) + { + disconnectObjectHelper(toDisconnect, 5); + // refresh(toDisconnect); + // refresh(from); + } + } /** * Replaces a deleted object @@ -2024,6 +2081,19 @@ public final class DataStore return synchronizedCommand(cmd, _dummy); } + public boolean queryServerSpiritState() + { + DataElement spirittype = findObjectDescriptor(IDataStoreConstants.DATASTORE_SPIRIT_DESCRIPTOR); + if (spirittype == null) return false; + DataElement cmd = localDescriptorQuery(spirittype, IDataStoreConstants.C_START_SPIRIT, 2); + if (cmd == null) return false; + + DataElement status = synchronizedCommand(cmd, _dummy); + if ((status != null) && status.getName().equals(DataStoreResources.model_done)) + return true; + else return false; + } + public DataElement queryHostJVM() { DataElement cmd = findCommandDescriptor(DataStoreSchema.C_QUERY_JVM); @@ -3361,11 +3431,9 @@ public final class DataStore { _tracingOn = false; } - - String logDir = getUserPreferencesDirectory(); if (_tracingOn) { - + String logDir = getUserPreferencesDirectory(); _traceFileHandle = new File(logDir, ".dstoreTrace"); try @@ -3377,7 +3445,7 @@ public final class DataStore { } } - + //_remoteClassLoader = new RemoteClassLoader(this); _classReqRepository = new HashMap(); @@ -3392,7 +3460,43 @@ public final class DataStore registerLocalClassLoader(this.getClass().getClassLoader()); } + + public void startDataElementRemoverThread() + { + if (!isVirtual() && _deRemover == null) + { + String memLogging = System.getProperty("DSTORE_MEMLOGGING_ON"); + _memLoggingOn = (memLogging != null && memLogging.equals("true")); + + if (_memLoggingOn) + { + String logDir = getUserPreferencesDirectory(); + _memLoggingFileHandle = new File(logDir, ".dstoreMemLogging"); + + try + { + _memLogFile = new RandomAccessFile(_memLoggingFileHandle, "rw"); + startMemLogging(); + } + catch (IOException e) + { + } + } + _deRemover = new DataElementRemover(this); + _deRemover.start(); + } + } + public boolean isDoSpirit() + { + if (isVirtual()) return _spiritModeOn; + else return _spiritModeOn && _spiritCommandReceived; + } + + public void receiveStartSpiritCommand() + { + _spiritCommandReceived = true; + } public IByteStreamHandler getDefaultByteStreamHandler() { @@ -3627,6 +3731,23 @@ public final class DataStore } } + private void disconnectObjectHelper(DataElement toDisconnect, int depth) + { + if (depth > 0) + { + depth--; + _deRemover.addToQueueForRemoval(toDisconnect); + for (int i = 0; i < toDisconnect.getNestedSize(); i++) + { + DataElement subDisconnect = toDisconnect.get(i); + if (subDisconnect != null && subDisconnect.getDataStore() == this && !subDisconnect.isSpirit()) + { + disconnectObjectHelper(subDisconnect, depth); + } + } + } + } + private String makeIdUnique(String id) { @@ -3693,6 +3814,28 @@ public final class DataStore trace("Start Tracing at " + System.currentTimeMillis()); } } + + public void startMemLogging() + { + if (_memLoggingOn && _memLogFile != null && _memLoggingFileHandle != null) + { + try + { + _memLogFile.seek(_memLoggingFileHandle.length()); + } + catch (IOException e) + { + } + + memLog("-----------------------------------------"); + memLog("Start Memory Logging at " + System.currentTimeMillis()); + } + } + + public void memLog(String str) + { + internalMemLog(str); + } public void trace(String str) { @@ -3739,6 +3882,22 @@ public final class DataStore } } } + + private void internalMemLog(String message) + { + if (_memLoggingOn && _memLogFile != null && message != null) + { + try + { + _memLogFile.writeBytes((new Date()).toString() + ": "); + _memLogFile.writeBytes(message); + _memLogFile.writeBytes(System.getProperty("line.separator")); + } + catch (IOException e) + { + } + } + } public void finish() { @@ -3988,5 +4147,24 @@ public final class DataStore _updateHandler.sendKeepAliveConfirmation(); } } + + /** + * @return what type of attribute tag is used on the peer DataStore to indicate whether dataelements + * are references, values, or spirit elements. If the peer DataStore is an older one, this will return + * "isRef", if its up-to-date, it will return "refType", and if the tag hasnt been determined yet, this method + * will return null. + */ + public String getReferenceTag() + { + return referenceTag; + } + /** + * Sets what type of attribute tag is used on the peer DataStore to indicate whether dataelements + * are references, values, or spirit elements. + */ + public void setReferenceTag(String tag) + { + referenceTag = tag; + } } \ No newline at end of file diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStoreResources.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStoreResources.java index 181295389f7..f4bba44f928 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStoreResources.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStoreResources.java @@ -109,4 +109,7 @@ public class DataStoreResources public static String DELETED="deleted"; public static String KEEPALIVE_TYPE="KEEPALIVE"; public static String KEEPALIVECONFIRM_TYPE="CONFIRMKEEPALIVE"; -} \ No newline at end of file + public static String REFERENCE="reference"; + public static String VALUE="value"; + public static String SPIRIT="spirit"; +} diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/Handler.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/Handler.java index 2bd3383a134..4fa6beb0e35 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/Handler.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/Handler.java @@ -27,7 +27,7 @@ public abstract class Handler extends Thread protected int _waitIncrement; protected DataStore _dataStore; - private boolean _keepRunning; + protected boolean _keepRunning; /** * Constructor diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/IDataStoreConstants.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/IDataStoreConstants.java index bd05b8f4ac4..2b9a44a5eff 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/IDataStoreConstants.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/IDataStoreConstants.java @@ -25,4 +25,7 @@ public interface IDataStoreConstants public static final String UNKNOWN_PROBLEM = "unknown problem connecting to server"; public static final String SERVER_FAILURE = "server failure: "; public static final String ATTEMPT_RECONNECT = "attempt reconnect"; + + public static final String DATASTORE_SPIRIT_DESCRIPTOR = "datastore.spirit"; + public static final String C_START_SPIRIT = "C_START_SPIRIT"; } diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java index 4b4c27352b7..054126f3025 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java @@ -20,7 +20,7 @@ import java.util.ArrayList; import java.util.List; import org.eclipse.dstore.core.java.IRemoteClassInstance; - +import org.eclipse.dstore.core.util.DataElementRemover; /** *

@@ -62,15 +62,11 @@ public abstract class UpdateHandler extends Handler protected void clean(DataElement object) { - if (_dataObjects.size() == 0) - { - clean(object, 2); - } + clean(object, 2); } protected synchronized void clean(DataElement object, int depth) { - if ((depth > 0) && (object != null) && object.getNestedSize() > 0) { List deletedList = _dataStore.findDeleted(object); @@ -81,6 +77,7 @@ public abstract class UpdateHandler extends Handler if (child != null && child.isDeleted()) { DataElement parent = child.getParent(); + if (child.isSpirit()) DataElementRemover.addToRemovedCount(); child.clear(); if (parent != null) { diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/Server.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/Server.java index bb1fee65f89..50bcbcd56b4 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/Server.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/Server.java @@ -37,6 +37,8 @@ public class Server */ public static void main(String[] args) { + //Tell the Launcher that we are starting + System.err.println(ServerReturnCodes.RC_DSTORE_SERVER_MAGIC); String jversion = System.getProperty("java.version"); diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerCommandHandler.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerCommandHandler.java index 47b0d99b042..9ad63c587ff 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerCommandHandler.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerCommandHandler.java @@ -29,6 +29,7 @@ import org.eclipse.dstore.core.model.DataStore; import org.eclipse.dstore.core.model.DataStoreAttributes; import org.eclipse.dstore.core.model.DataStoreResources; import org.eclipse.dstore.core.model.DataStoreSchema; +import org.eclipse.dstore.core.model.IDataStoreConstants; /** * The ServerCommandHandler is reponsible for maintaining @@ -224,6 +225,7 @@ public class ServerCommandHandler extends CommandHandler clientTicket.setAttribute(DE.A_VALUE,DataStoreResources.model_invalid); } _dataStore.update(clientTicket); + _dataStore.startDataElementRemoverThread(); status.setAttribute(DE.A_NAME,DataStoreResources.model_done); } else if (commandName.equals(DataStoreSchema.C_SET)) @@ -330,10 +332,13 @@ public class ServerCommandHandler extends CommandHandler _dataStore.refresh(schemaRoot); status.setAttribute(DE.A_NAME,DataStoreResources.model_done); } - else if (_dataStore.validTicket()) + else if (commandName.equals(IDataStoreConstants.C_START_SPIRIT)) + { + _dataStore.receiveStartSpiritCommand(); + status.setAttribute(DE.A_NAME, DataStoreResources.model_done); + } + else if (_dataStore.validTicket() && _minerLoader != null) { - - if (status != null) { boolean failure = false; diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerLauncher.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerLauncher.java index 2d1d863c408..c5d88e19a23 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerLauncher.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerLauncher.java @@ -247,6 +247,8 @@ public class ServerLauncher extends Thread + timeout + " " + ticket + + " " //$NON-NLS-1$ + + System.getProperty("java.home"); //$NON-NLS-1$ ; String[] authArray = { "sh", "-c", authString }; @@ -290,11 +292,20 @@ public class ServerLauncher extends Thread } else { + //read over stuff coming from the login shell invocation String status = _errReader.readLine(); - _port = _errReader.readLine(); - + while (status!=null && !status.equals(ServerReturnCodes.RC_DSTORE_SERVER_MAGIC)) + { + status = _errReader.readLine(); + } + //now read the real server status + if (status != null) + { + status = _errReader.readLine(); + } if ((status != null) && status.equals(ServerReturnCodes.RC_SUCCESS)) { + _port = _errReader.readLine(); _errReader.readLine(); _writer.println(IDataStoreConstants.CONNECTED); _writer.println(_port); @@ -309,7 +320,7 @@ public class ServerLauncher extends Thread { status = new String(IDataStoreConstants.UNKNOWN_PROBLEM); } - + //TODO Make sure that the client doesnt try connecting forever _writer.println(status); _serverProcess.destroy(); diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerReturnCodes.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerReturnCodes.java index cf8d9317585..8b4b6c4bcde 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerReturnCodes.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerReturnCodes.java @@ -22,7 +22,7 @@ package org.eclipse.dstore.core.server; */ public class ServerReturnCodes { - + public static final String RC_DSTORE_SERVER_MAGIC = "Dstore Server Starting..."; //$NON-NLS-1$ public static final String RC_SUCCESS = "Server Started Successfully"; diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerSSLProperties.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerSSLProperties.java index 2bd55a0932a..7acdae6a70b 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerSSLProperties.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerSSLProperties.java @@ -48,21 +48,21 @@ public class ServerSSLProperties implements ISSLProperties ResourceBundle properties = ResourceBundle.getBundle("ssl"); if (properties != null) { - _enableSSL = properties.getString(ENABLE_SSL).equals("true"); + _enableSSL = properties.getString(ENABLE_SSL).trim().equals("true"); if (_enableSSL) { try { - _daemonKeyStorePath = properties.getString(DAEMON_KEYSTORE_FILE); - _daemonKeyStorePassword = properties.getString(DAEMON_KEYSTORE_PASSWORD); + _daemonKeyStorePath = properties.getString(DAEMON_KEYSTORE_FILE).trim(); + _daemonKeyStorePassword = properties.getString(DAEMON_KEYSTORE_PASSWORD).trim(); } catch (Exception e) { } try { - _serverKeyStorePath = properties.getString(SERVER_KEYSTORE_FILE); - _serverKeyStorePassword = properties.getString(SERVER_KEYSTORE_PASSWORD); + _serverKeyStorePath = properties.getString(SERVER_KEYSTORE_FILE).trim(); + _serverKeyStorePassword = properties.getString(SERVER_KEYSTORE_PASSWORD).trim(); } catch (Exception e) { diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/CommandGenerator.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/CommandGenerator.java index 3fec222e167..73cb06809f4 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/CommandGenerator.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/CommandGenerator.java @@ -151,7 +151,7 @@ public class CommandGenerator commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName()); - if (dataObject.isUpdated()) + if (dataObject.isUpdated() && !dataObject.isSpirit()) { _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_contents); } @@ -168,7 +168,7 @@ public class CommandGenerator DataElement arg = (DataElement) arguments.get(i); if (arg != null) { - if (!arg.isUpdated()) + if (!arg.isUpdated() || arg.isSpirit()) { commandObject.addNestedData(arg, false); } @@ -208,7 +208,7 @@ public class CommandGenerator DataElement tempRoot = _dataStore.getTempRoot(); commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName()); clearDeleted(dataObject); - if (refArg || dataObject.isUpdated()) + if ((refArg || dataObject.isUpdated()) && !dataObject.isSpirit()) { _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_contents); } @@ -218,7 +218,7 @@ public class CommandGenerator commandObject.addNestedData(dataObject, false); } - if (!arg.isUpdated()) + if (!arg.isUpdated() || arg.isSpirit()) { commandObject.addNestedData(arg, false); } @@ -254,7 +254,7 @@ public class CommandGenerator commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName()); clearDeleted(dataObject); - if (refArg || dataObject.isUpdated()) + if ((refArg || dataObject.isUpdated()) && !dataObject.isSpirit()) { _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_arguments); } diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/DataElementRemover.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/DataElementRemover.java new file mode 100644 index 00000000000..6deccb6dd8c --- /dev/null +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/DataElementRemover.java @@ -0,0 +1,179 @@ +package org.eclipse.dstore.core.util; + +import java.util.LinkedList; + +import org.eclipse.dstore.core.model.DataElement; +import org.eclipse.dstore.core.model.DataStore; +import org.eclipse.dstore.core.model.Handler; +import org.eclipse.dstore.core.model.IDataStoreConstants; + +public class DataElementRemover extends Handler +{ + private LinkedList _queue; + private static int numRemoved = 0; + private static int numDisconnected = 0; + private static int numCreated = 0; + private static int numGCed = 0; + + // The following determine how DataElements are chosen to be removed once they + // are in the queue for removal. + // The queue is checked every _intervalTime milliseconds and all elements + // that are older than _expiryTime milliseconds are removed. + public static final int DEFAULT_EXPIRY_TIME = 600; // in seconds + public static final int DEFAULT_INTERVAL_TIME = 60; // in seconds + private int _intervalTime = DEFAULT_INTERVAL_TIME * 1000; + private int _expiryTime = DEFAULT_EXPIRY_TIME * 1000; + public static final String EXPIRY_TIME_PROPERTY_NAME = "SPIRIT_EXPIRY_TIME"; + public static final String INTERVAL_TIME_PROPERTY_NAME = "SPIRIT_INTERVAL_TIME"; + + public DataElementRemover(DataStore dataStore) + { + super(); + _dataStore = dataStore; + _queue = new LinkedList(); + getTimes(); + setWaitTime(_intervalTime); + DataElement spiritnode = _dataStore.createObjectDescriptor(_dataStore.getDescriptorRoot(), IDataStoreConstants.DATASTORE_SPIRIT_DESCRIPTOR); + _dataStore.createCommandDescriptor(spiritnode, "StartSpirit", "DataElementRemover", IDataStoreConstants.C_START_SPIRIT); + _dataStore.refresh(_dataStore.getDescriptorRoot()); + } + + protected void getTimes() + { + try + { + String expiryTime = System.getProperty(EXPIRY_TIME_PROPERTY_NAME); + if (expiryTime != null && !expiryTime.equals("")) _expiryTime = Integer.parseInt(expiryTime) * 1000; + } + catch (Exception e) + { + System.out.println("Invalid spirit expiry time property, using default."); + _expiryTime = DEFAULT_EXPIRY_TIME; + } + try + { + String intervalTime = System.getProperty(INTERVAL_TIME_PROPERTY_NAME); + if (intervalTime != null && !intervalTime.equals("")) _intervalTime = Integer.parseInt(intervalTime) * 1000; + } + catch (Exception e) + { + System.out.println("Invalid spirit interval time property, using default."); + _intervalTime = DEFAULT_INTERVAL_TIME; + } + } + + public static void addToRemovedCount() + { + numRemoved++; + } + + public static void addToCreatedCount() + { + numCreated++; + } + + public static void addToGCedCount() + { + numGCed++; + } + + public synchronized void addToQueueForRemoval(DataElement element) + { + synchronized (_queue) + { + if (_dataStore.isDoSpirit() && _dataStore == element.getDataStore()) + { + QueueItem item = new QueueItem(element, System.currentTimeMillis()); + _queue.add(item); + } + } + } + + public void handle() + { + clearQueue(); + } + + public synchronized void clearQueue() + { + synchronized (_queue) + { + _dataStore.memLog(" "); + int disconnected = 0; + if (!_dataStore.isDoSpirit()) + { + if (_queue.size() > 0) + { + _dataStore.memLog("Clearing queue of size " + _queue.size() + ". DSTORE_SPIRIT_ON not set or set to false."); + _queue.clear(); + } + _dataStore.memLog("Total heap size: " + Runtime.getRuntime().totalMemory()); + _dataStore.memLog("Elements created so far: " + numCreated); + _dataStore.memLog("Elements disconnected so far: " + numDisconnected); + _dataStore.memLog("Spirit elements cleaned so far: " + numRemoved); + _dataStore.memLog("DataElements GCed so far: " + numGCed); + return; + } + _dataStore.memLog("Total heap size before disconnection: " + Runtime.getRuntime().totalMemory()); + + _dataStore.memLog("Size of queue: " + _queue.size()); + + while (_queue.size() > 0 && System.currentTimeMillis() - ((QueueItem) _queue.getFirst()).timeStamp > _expiryTime) + { + DataElement toBeDisconnected = ((QueueItem) _queue.removeFirst()).dataElement; + if (!toBeDisconnected.isSpirit()) + { + toBeDisconnected.setSpirit(true); + _dataStore.refresh(toBeDisconnected); + disconnected++; + numDisconnected++; + } + else + { + //_dataStore.memLog(toBeDisconnected.toString()); + } + _dataStore.getHashMap().remove(toBeDisconnected.getId()); + } + _dataStore.memLog("Disconnected " + disconnected + " DataElements."); + _dataStore.memLog("Elements created so far: " + numCreated); + _dataStore.memLog("Elements disconnected so far: " + numDisconnected); + _dataStore.memLog("Spirit elements cleaned so far: " + numRemoved); + _dataStore.memLog("DataElements GCed so far: " + numGCed); + } + } + + protected class QueueItem + { + public DataElement dataElement; + public long timeStamp; + + public QueueItem(DataElement element, long stamp) + { + dataElement = element; + timeStamp = stamp; + } + } + + /** + * Runs the handler loop in a thread. + */ + public void run() + { + while (_keepRunning) + { + try + { + Thread.sleep(_waitIncrement); + Thread.yield(); + } + catch (InterruptedException e) + { + e.printStackTrace(); + finish(); + return; + } + handle(); + } + } +} + diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/XMLgenerator.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/XMLgenerator.java index 36f322fd319..da59e1e8bea 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/XMLgenerator.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/XMLgenerator.java @@ -17,8 +17,6 @@ package org.eclipse.dstore.core.util; import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.ObjectOutputStream; import java.io.PipedInputStream; @@ -290,6 +288,27 @@ public class XMLgenerator } } + + private void addReferenceTypeAttribute(DataElement object) + { + if (object.isSpirit()) + { + addAttribute(DE.P_REF_TYPE, DataStoreResources.SPIRIT); + } + else + { + if (object.isReference()) + { + if (_dataStore.getReferenceTag() != null && _dataStore.getReferenceTag().equals(DE.P_REF_TYPE)) addAttribute(DE.P_REF_TYPE, DataStoreResources.REFERENCE); + else addAttribute(DE.P_REF_TYPE, DataStoreResources.TRUE); + } + else + { + if (_dataStore.getReferenceTag() != null && _dataStore.getReferenceTag().equals(DE.P_REF_TYPE)) addAttribute(DE.P_REF_TYPE, DataStoreResources.VALUE); + else addAttribute(DE.P_REF_TYPE, DataStoreResources.FALSE); + } + } + } private void addFile(byte[] bytes, int size, boolean binary) { @@ -500,16 +519,9 @@ public class XMLgenerator addAttribute(DE.P_NAME, object.getAttribute(DE.A_NAME)); addAttribute(DE.P_VALUE, object.getAttribute(DE.A_VALUE)); addAttribute(DE.P_SOURCE, object.getAttribute(DE.A_SOURCE)); - addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION)); - - if (object.isReference()) - { - addAttribute(DE.P_ISREF, DataStoreResources.TRUE); - } - else - { - addAttribute(DE.P_ISREF, DataStoreResources.FALSE); - } + addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION)); + + addReferenceTypeAttribute(object); addAttribute(DE.P_DEPTH, "" + size); addFile(bytes, size, binary); @@ -537,15 +549,9 @@ public class XMLgenerator addAttribute(DE.P_NAME, object.getAttribute(DE.A_NAME)); addAttribute(DE.P_VALUE, object.getAttribute(DE.A_VALUE)); addAttribute(DE.P_SOURCE, object.getAttribute(DE.A_SOURCE)); - addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION)); - if (object.isReference()) - { - addAttribute(DE.P_ISREF, "true"); - } - else - { - addAttribute(DE.P_ISREF, "false"); - } + addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION)); + + addReferenceTypeAttribute(object); addAttribute(DE.P_DEPTH, "" + size); addFile(bytes, size, true); @@ -584,16 +590,9 @@ public class XMLgenerator addAttribute(DE.P_NAME, object.getAttribute(DE.A_NAME)); addAttribute(DE.P_VALUE, object.getAttribute(DE.A_VALUE)); addAttribute(DE.P_SOURCE, object.getAttribute(DE.A_SOURCE)); - addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION)); - - if (object.isReference()) - { - addAttribute(DE.P_ISREF, "true"); - } - else - { - addAttribute(DE.P_ISREF, "false"); - } + addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION)); + + addReferenceTypeAttribute(object); addAttribute(DE.P_DEPTH, "" + object.depth()); addData(object.getBuffer()); @@ -625,16 +624,9 @@ public class XMLgenerator addAttribute(DE.P_NAME, object.getAttribute(DE.A_NAME)); addAttribute(DE.P_VALUE, object.getAttribute(DE.A_VALUE)); addAttribute(DE.P_SOURCE, object.getAttribute(DE.A_SOURCE)); - addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION)); - - if (object.isReference()) - { - addAttribute(DE.P_ISREF, "true"); - } - else - { - addAttribute(DE.P_ISREF, "false"); - } + addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION)); + + addReferenceTypeAttribute(object); _state = BODY; endTag(tagType); } @@ -652,16 +644,9 @@ public class XMLgenerator addAttribute(DE.P_NAME, object.getAttribute(DE.A_NAME)); addAttribute(DE.P_VALUE, object.getAttribute(DE.A_VALUE)); addAttribute(DE.P_SOURCE, object.getAttribute(DE.A_SOURCE)); - addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION)); - - if (object.isReference()) - { - addAttribute(DE.P_ISREF, "true"); - } - else - { - addAttribute(DE.P_ISREF, "false"); - } + addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION)); + + addReferenceTypeAttribute(object); try { diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/XMLparser.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/XMLparser.java index ac1e486de61..6128091167d 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/XMLparser.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/XMLparser.java @@ -473,6 +473,12 @@ public class XMLparser if (xmlTag != null) { String trimmedTag = xmlTag.trim(); + + if (_dataStore.getReferenceTag() == null) + { + if (trimmedTag.indexOf(DE.P_ISREF + "=") > -1) _dataStore.setReferenceTag(DE.P_ISREF); + else if (trimmedTag.indexOf(DE.P_REF_TYPE + "=") > -1) _dataStore.setReferenceTag(DE.P_REF_TYPE); + } if (!_tagStack.empty()) { @@ -752,9 +758,11 @@ public class XMLparser } else { - String isRefStr = attributes[DE.A_ISREF]; + String refType = attributes[DE.A_REF_TYPE]; + boolean isSpirit = false; + if (refType != null) isSpirit = refType.equals(DataStoreResources.SPIRIT); - if ((isRefStr != null) && isRefStr.equals("true")) + if ((refType != null) && (refType.equals(DataStoreResources.TRUE) || refType.equals(DataStoreResources.REFERENCE))) { // new reference String origId = attributes[DE.A_NAME]; @@ -806,6 +814,15 @@ public class XMLparser } else { + if (isSpirit) + { + if (!_dataStore.isVirtual()) attributes[DE.A_REF_TYPE] = DataStoreResources.VALUE; + result.setSpirit(_dataStore.isVirtual()); + } + else + { + result.setSpirit(false); + } result.setAttributes(attributes); } @@ -852,8 +869,25 @@ public class XMLparser } else { - // new object - result = _dataStore.createObject(parent, attributes); + // new object + if (_dataStore.isVirtual() && parent != null) + { + result = _dataStore.find(parent, DE.A_NAME, attributes[DE.A_NAME], 1); + if (result != null && result.getValue().equals(attributes[DE.A_VALUE]) && result.isSpirit()) + _dataStore.deleteObject(parent, result); + } + if (isSpirit) + { + if (!_dataStore.isVirtual()) attributes[DE.A_REF_TYPE] = DataStoreResources.VALUE; + result = _dataStore.createObject(parent, attributes); + result.setSpirit(_dataStore.isVirtual()); + } + else + { + result = _dataStore.createObject(parent, attributes); + result.setSpirit(false); + } + } diff --git a/rse/plugins/org.eclipse.rse.connectorservice.dstore/src/org/eclipse/rse/connectorservice/dstore/DStoreConnectorService.java b/rse/plugins/org.eclipse.rse.connectorservice.dstore/src/org/eclipse/rse/connectorservice/dstore/DStoreConnectorService.java index dac7d98380f..5a3e11c91d9 100644 --- a/rse/plugins/org.eclipse.rse.connectorservice.dstore/src/org/eclipse/rse/connectorservice/dstore/DStoreConnectorService.java +++ b/rse/plugins/org.eclipse.rse.connectorservice.dstore/src/org/eclipse/rse/connectorservice/dstore/DStoreConnectorService.java @@ -767,6 +767,8 @@ public class DStoreConnectorService extends AbstractConnectorService implements { dataStore.showTicket(null); } + + if (dataStore.isDoSpirit()) dataStore.queryServerSpiritState(); // Fire comm event to signal state changed fireCommunicationsEvent(CommunicationsEvent.AFTER_CONNECT); @@ -818,7 +820,7 @@ public class DStoreConnectorService extends AbstractConnectorService implements statusMonitor.waitForUpdate(schemaStatus); statusMonitor.waitForUpdate(initStatus); } - + long t2 = System.currentTimeMillis(); System.out.println("connect time = "+(t2 - t1)); diff --git a/rse/plugins/org.eclipse.rse.eclipse.filesystem/src/org/eclipse/rse/eclipse/filesystem/RSEFileSystem.java b/rse/plugins/org.eclipse.rse.eclipse.filesystem/src/org/eclipse/rse/eclipse/filesystem/RSEFileSystem.java index 467b521d518..73d3cc7480e 100644 --- a/rse/plugins/org.eclipse.rse.eclipse.filesystem/src/org/eclipse/rse/eclipse/filesystem/RSEFileSystem.java +++ b/rse/plugins/org.eclipse.rse.eclipse.filesystem/src/org/eclipse/rse/eclipse/filesystem/RSEFileSystem.java @@ -17,6 +17,7 @@ package org.eclipse.rse.eclipse.filesystem; import java.net.URI; +import java.rmi.RemoteException; import java.util.HashMap; import org.eclipse.core.filesystem.IFileStore; @@ -27,6 +28,7 @@ import org.eclipse.rse.model.ISystemRegistry; import org.eclipse.rse.subsystems.files.core.model.RemoteFileUtility; import org.eclipse.rse.subsystems.files.core.subsystems.IRemoteFile; import org.eclipse.rse.subsystems.files.core.subsystems.IRemoteFileSubSystem; +import org.eclipse.rse.subsystems.files.core.subsystems.RemoteFileEmpty; import org.eclipse.rse.ui.RSEUIPlugin; import org.eclipse.swt.widgets.Shell; import org.eclipse.ui.PlatformUI; @@ -160,6 +162,7 @@ public class RSEFileSystem extends FileSystem catch (Exception e) { e.printStackTrace(); + return FileStoreConversionUtility.convert(null, new RemoteFileEmpty()); } return null; } diff --git a/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/command/CommandMinerThread.java b/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/command/CommandMinerThread.java index 05841f01d96..0efb704529c 100644 --- a/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/command/CommandMinerThread.java +++ b/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/command/CommandMinerThread.java @@ -361,6 +361,7 @@ public class CommandMinerThread extends MinerThread status.setAttribute(DE.A_NAME, "progress"); _dataStore.update(status); + _dataStore.disconnectObjects(status); _stdOutputHandler = new OutputHandler(_stdInput, null, _isWindows || _isTTY, false, _isShell, this); _stdOutputHandler.setWaitTime(10); _stdOutputHandler.start(); @@ -1005,6 +1006,7 @@ public class CommandMinerThread extends MinerThread } _dataStore.refresh(_status); + _dataStore.disconnectObjects(_status); } public void createPrompt(String line, String fileName) @@ -1231,6 +1233,7 @@ public class CommandMinerThread extends MinerThread obj.setAttribute(DE.A_SOURCE, obj.getSource() + ':' + line.toString()); } _dataStore.refresh(_status); + _dataStore.disconnectObjects(_status); return obj; } else diff --git a/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/command/QueryPathThread.java b/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/command/QueryPathThread.java index f68e96c88ba..e3705872bcd 100644 --- a/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/command/QueryPathThread.java +++ b/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/command/QueryPathThread.java @@ -126,6 +126,7 @@ public class QueryPathThread extends Thread } status.setAttribute(DE.A_NAME, "done"); _dataStore.refresh(status); + _dataStore.disconnectObjects(status); } private void resolveCommandsInPath(File file, DataElement status) diff --git a/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/filesystem/FileClassifier.java b/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/filesystem/FileClassifier.java index 6a70f400f3e..29d3bd076e0 100644 --- a/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/filesystem/FileClassifier.java +++ b/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/filesystem/FileClassifier.java @@ -302,6 +302,7 @@ public class FileClassifier extends Thread } } } + _dataStore.disconnectObject(_subject); } /**