[390037] [dstore] Duplicated items in the System view

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java

@@ -17,6 +17,7 @@
  * David McKnight   (IBM) - [380158] [dstore] DataStore.command() fails when multiple commands issue simultaneously
  * David McKnight   (IBM) - [385793] [dstore] DataStore spirit mechanism and other memory improvements needed
  * David McKnight   (IBM) - [389286] [dstore] element delete should not clear _attributes since elements get recycled
+ * David McKnight   (IBM) - [390037] [dstore] Duplicated items in the System view
  *******************************************************************************/
 
 package org.eclipse.dstore.core.model;
@@ -885,6 +886,15 @@ public final class DataElement implements IDataElement
 	public void setSpirit(boolean flag)
 	{
 		_isSpirit = flag;
+		String refType = getAttribute(DE.A_REF_TYPE);
+		if (refType != null){
+			if (_isSpirit && !refType.equals(DataStoreResources.SPIRIT)) {
+				setAttribute(DE.A_REF_TYPE, DataStoreResources.SPIRIT);
+			}
+			else if (refType.equals(DataStoreResources.SPIRIT)){ // if it was a spirit, change it back
+				setAttribute(DE.A_REF_TYPE, DataStoreResources.VALUE);
+			}
+		}
 	}
 
 	/**
@@ -1557,7 +1567,6 @@ public final class DataElement implements IDataElement
 
 	private void initialize(DataElement typeDescriptor)
 	{
-		getAttributes(); // make sure attributes is not null
 		_isReference = false;
 		_isDescriptor = false;
 		_depth = 1;
@@ -1659,10 +1668,10 @@ public final class DataElement implements IDataElement
 		{
 			// set delete attribute
 			
-			setAttribute(DE.A_SOURCE, null);
+			setAttribute(DE.A_SOURCE, ""); //$NON-NLS-1$
 			setAttribute(DE.A_VALUE, DataStoreResources.DELETED);
-			setAttribute(DE.A_TYPE, null);
+			setAttribute(DE.A_TYPE, ""); //$NON-NLS-1$
-			setAttribute(DE.A_NAME, null);
+			setAttribute(DE.A_NAME, ""); //$NON-NLS-1$
 			
 			_isUpdated = false;
 			_isExpanded = true;

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java

@@ -44,6 +44,7 @@
  * David McKnight   (IBM) - [373507] [dstore][multithread] reduce heap memory on disconnect for server
  * David McKnight   (IBM) - [385097] [dstore] DataStore spirit mechanism is not enabled
  * David McKnight   (IBM) - [385793] [dstore] DataStore spirit mechanism and other memory improvements needed
+ * David McKnight   (IBM) - [390037] [dstore] Duplicated items in the System view
  *******************************************************************************/
 
 package org.eclipse.dstore.core.model;
@@ -3797,6 +3798,13 @@ public final class DataStore
 			{
 				String logDir = getUserPreferencesDirectory();
 				_memLoggingFileHandle = new File(logDir, ".dstoreMemLogging"); //$NON-NLS-1$
+				// need this check, otherwise, we don't create this log file
+				if (!_memLoggingFileHandle.exists()){
+					try { // try to create it
+						_memLoggingFileHandle.createNewFile();
+					} catch (IOException e) {
+					}
+				}
 				if (_memLoggingFileHandle.canWrite()){
 					try
 					{

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java

@@ -16,6 +16,7 @@
  * David McKnight   (IBM) - [226561] [apidoc] Add API markup to RSE Javadocs where extend / implement is allowed
  * David McKnight   (IBM) - [385793] [dstore] DataStore spirit mechanism and other memory improvements needed
  * David McKnight   (IBM) - [389286] [dstore] element delete should not clear _attributes since elements get recycled
+ * David McKnight   (IBM) - [390037] [dstore] Duplicated items in the System view
  *******************************************************************************/
 
 package org.eclipse.dstore.core.model;
@@ -90,17 +91,16 @@ public abstract class UpdateHandler extends Handler
 
 					cleanChildren(child); // clean the children
 
-					boolean virtual = _dataStore.isVirtual();
+
 					if (child.isSpirit())
 					{
-						if (!virtual){ // leave the client copy
+						// officially delete this now 
-							// officially delete this now
+						// will only happen on server since, on client, 
-							child.delete();
+						// the above call to isDeleted() returns false for spirited
-						}
+						child.delete();						
-					}
-					if (!virtual){ // leave the client attributes if spirited
-						child.clear();
 					}
+
+					child.clear();
 					if (parent != null)
 					{
 						synchronized (parent)
@@ -128,20 +128,22 @@ public abstract class UpdateHandler extends Handler
 		List nestedData = parent.getNestedData();
 		if (nestedData != null)
 		{
-			boolean isVirtual = parent.getDataStore().isVirtual();
+			synchronized (nestedData){
-		for (int i = 0; i < nestedData.size(); i++){
+				for (int i = nestedData.size() - 1; i >= 0; i--){
-			DataElement child = (DataElement)nestedData.get(i);
+					DataElement child = (DataElement)nestedData.get(i);
-			cleanChildren(child);
+					cleanChildren(child);
-
+	
-			if (!isVirtual && child.isSpirit())
+					if (child.isSpirit())
-			{
+					{
-				// officially delete this now
+						// officially delete this now
-				child.delete();
+						child.delete();
+					}
+				
+					child.clear();
+					parent.removeNestedData(child);
+				}
 			}
-			child.clear();
+		}	
-			parent.removeNestedData(child);
-		}
-		}
 	}
 
 	/**
@@ -174,14 +176,17 @@ public abstract class UpdateHandler extends Handler
 	{
 		synchronized (_dataObjects)
 		{
+			if (object != null){
+				String type = object.getType();
+				boolean isStatus = DataStoreResources.model_status.equals(type);
 			if (immediate)
 			{
 				_dataObjects.add(0, object);
 				// none of this immediate stuff - just put it at the beginning
 				//handle();
 			}
-			else
+			else 
-			{
+			{				
 				if (!_dataObjects.contains(object))
 				{
 						_dataObjects.add(object);
@@ -191,7 +196,7 @@ public abstract class UpdateHandler extends Handler
 
 					if (_dataStore != null && object != null && !object.isDeleted())
 					{
-						if (object.getType().equals(DataStoreResources.model_status))
+						if (isStatus)
 						{
 							if (object.getName().equals(DataStoreResources.model_done))
 							{
@@ -202,10 +207,16 @@ public abstract class UpdateHandler extends Handler
 								// object does not come back to client (as "done") before the results of a query
 								_dataObjects.remove(object);
 								_dataObjects.add(object);
+								
 							}
 						}
 					}
 				}
+			}
+				if (_dataStore != null && !_dataStore.isVirtual() && isStatus){
+					_dataStore.disconnectObjects(object.getParent()); // spirit the command
+				}
+			
 			}
 		}
 		notifyInput();

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/CommandGenerator.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2002, 2008 IBM Corporation and others.
+ * Copyright (c) 2002, 2012 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -14,6 +14,7 @@
  * Contributors:
  * David McKnight   (IBM) - [225507][api][breaking] RSE dstore API leaks non-API types
  * David McKnight   (IBM) - [226561] [apidoc] Add API markup to RSE Javadocs where extend / implement is allowed
+ * David McKnight   (IBM) - [390037] [dstore] Duplicated items in the System view
  *******************************************************************************/
 
 package org.eclipse.dstore.core.util;
@@ -156,8 +157,15 @@ public class CommandGenerator
             }
             else
             {
-            	dataObject.setPendingTransfer(true);
+               	dataObject.setPendingTransfer(true);
-                commandObject.addNestedData(dataObject, false);
+            	if (dataObject.isSpirit() && _dataStore.isVirtual()){
+            		// resurrecting spirited element
+            		dataObject.setSpirit(false);            		
+            		// clear out old data - otherwise, we can end up with duplicates
+            		dataObject.removeNestedData();
+            	}   
+            	
+                 commandObject.addNestedData(dataObject, false);
             }
 
             if (arguments != null)
@@ -213,6 +221,12 @@ public class CommandGenerator
             else
             {
             	dataObject.setPendingTransfer(true);
+               	if (dataObject.isSpirit() && _dataStore.isVirtual()){
+               		// resurrecting spirited element
+            		dataObject.setSpirit(false);            		
+            		// clear out old data - otherwise, we can end up with duplicates
+            		dataObject.removeNestedData();
+            	}   
                 commandObject.addNestedData(dataObject, false);
             }
 
@@ -259,6 +273,12 @@ public class CommandGenerator
             else
             {
             	dataObject.setPendingTransfer(true);
+               	if (dataObject.isSpirit() && _dataStore.isVirtual()){
+            		// resurrecting spirited element
+            		dataObject.setSpirit(false);            		
+            		// clear out old data - otherwise, we can end up with duplicates
+            		dataObject.removeNestedData();
+            	}   
                 commandObject.addNestedData(dataObject, false);
             }
 

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java

@@ -20,6 +20,7 @@
  *  David McKnight   (IBM) - [371401] [dstore][multithread] avoid use of static variables - causes memory leak after disconnect
  *  David McKnight   (IBM)  - [373507] [dstore][multithread] reduce heap memory on disconnect for server
  *  David McKnight   (IBM)  - [385097] [dstore] DataStore spirit mechanism is not enabled
+ *  David McKnight   (IBM) - [390037] [dstore] Duplicated items in the System view
  *******************************************************************************/
 
 package org.eclipse.dstore.internal.core.util;
@@ -31,24 +32,22 @@ import java.util.LinkedList;
 import org.eclipse.dstore.core.model.DataElement;
 import org.eclipse.dstore.core.model.DataStore;
 import org.eclipse.dstore.core.model.Handler;
-import org.eclipse.dstore.core.model.IDataStoreConstants;
+
 
 public class DataElementRemover extends Handler 
 {
 	private LinkedList _queue;
-	private static int numRemoved = 0;
 	private static int numDisconnected = 0;
-	private static int numCreated = 0;
 	//private int numGCed = 0;
 	
 	// The following determine how DataElements are chosen to be removed once they
 	// are in the queue for removal. 	
 	// The queue is checked every _intervalTime milliseconds and all elements
 	// that are older than _expiryTime milliseconds are removed.
-	public static final int DEFAULT_EXPIRY_TIME = 600; // in seconds
+	public static final int DEFAULT_EXPIRY_TIME = 60; // in seconds
 	public static final int DEFAULT_INTERVAL_TIME = 60; // in seconds
-	private int _intervalTime = DEFAULT_INTERVAL_TIME * 10;
+	private int _intervalTime = DEFAULT_INTERVAL_TIME * 1000;
-	private int _expiryTime = DEFAULT_EXPIRY_TIME * 10;
+	private int _expiryTime = DEFAULT_EXPIRY_TIME * 1000;
 	public static final String EXPIRY_TIME_PROPERTY_NAME = "SPIRIT_EXPIRY_TIME"; //$NON-NLS-1$
 	public static final String INTERVAL_TIME_PROPERTY_NAME = "SPIRIT_INTERVAL_TIME"; //$NON-NLS-1$
 	public MemoryManager _memoryManager;
@@ -73,7 +72,6 @@ public class DataElementRemover extends Handler
 		catch (Exception e)
 		{
 			System.out.println("Invalid spirit expiry time property, using default."); //$NON-NLS-1$
-			_expiryTime = DEFAULT_EXPIRY_TIME;
 		}
 		try
 		{
@@ -83,23 +81,21 @@ public class DataElementRemover extends Handler
 		catch (Exception e)
 		{
 			System.out.println("Invalid spirit interval time property, using default."); //$NON-NLS-1$
-			_intervalTime = DEFAULT_INTERVAL_TIME;
 		}
 	}
 	
 	public static void addToRemovedCount()
 	{
-		numRemoved++;
+		// not using this anymore - better to get this from DataStore
 	}
 	
 	public static void addToCreatedCount()
 	{
-		numCreated++;
+		// not using this anymore - better to get this from DataStore
 	}
 	
 	public static void addToGCedCount()
 	{
-		//numGCed++;
 	}
 
 	
@@ -148,10 +144,11 @@ public class DataElementRemover extends Handler
 					_queue.clear();
 				}
 				_dataStore.memLog("Total heap size: " + Runtime.getRuntime().totalMemory()); //$NON-NLS-1$
-				_dataStore.memLog("Elements created so far: " + numCreated); //$NON-NLS-1$
+				_dataStore.memLog("Live elements: " + _dataStore.getNumElements()); //$NON-NLS-1$
+				_dataStore.memLog("Recycled elements: " + _dataStore.getNumRecycled()); //$NON-NLS-1$
 				_dataStore.memLog("Elements disconnected so far: " + numDisconnected); //$NON-NLS-1$
-				_dataStore.memLog("Spirit elements cleaned so far: " + numRemoved); //$NON-NLS-1$
+
-			
+
 				// no longer a helpful stat since we no longer use finalize
 				// _dataStore.memLog("DataElements GCed so far: " + numGCed); //$NON-NLS-1$
 				return;
@@ -188,9 +185,9 @@ public class DataElementRemover extends Handler
 			_dataStore.refresh(toRefresh);
 			
 			_dataStore.memLog("Disconnected " + disconnected + " DataElements."); //$NON-NLS-1$ //$NON-NLS-2$
-			_dataStore.memLog("Elements created so far: " + numCreated); //$NON-NLS-1$
+			_dataStore.memLog("Live elements: " + _dataStore.getNumElements()); //$NON-NLS-1$
+			_dataStore.memLog("Recycled elements: " + _dataStore.getNumRecycled()); //$NON-NLS-1$
 			_dataStore.memLog("Elements disconnected so far: " + numDisconnected); //$NON-NLS-1$
-			_dataStore.memLog("Spirit elements cleaned so far: " + numRemoved); //$NON-NLS-1$
 			
 			// no longer a helpful stat since we no longer use finalize
 			// _dataStore.memLog("DataElements GCed so far: " + numGCed); //$NON-NLS-1$
@@ -229,7 +226,7 @@ public class DataElementRemover extends Handler
 		{
 			try
 			{
-				Thread.sleep(100000); // wait 100 seconds
+				Thread.sleep(_intervalTime);
 				Thread.yield();
 			}
 			catch (InterruptedException e)

rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/UniversalFileSystemMiner.java

@@ -44,6 +44,7 @@
  * David McKnight  (IBM)  - [dstore] cancelable threads not removed fast enough from Hashmap, resulting in OOM
  * David McKnight   (IBM) - [371401] [dstore][multithread] avoid use of static variables - causes memory leak after disconnect
  * Noriaki Takatsu  (IBM) - [380562] [multithread][dstore] File Search is not canceled by the client UI on disconnect
+ * David McKnight   (IBM)        - [390037] [dstore] Duplicated items in the System view
  *******************************************************************************/
 
 package org.eclipse.rse.dstore.universal.miners;
@@ -1121,6 +1122,7 @@ public class UniversalFileSystemMiner extends Miner {
 		}
 
 		_dataStore.refresh(subject);
+		_dataStore.disconnectObject(subject);
 		return statusDone(status);
 	}
 

rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2006, 2011 IBM Corporation and others.
+ * Copyright (c) 2006, 2012 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -65,6 +65,7 @@
  * David McKnight   (IBM)        - [308770] [dstore] Remote Search using old server fails with NPE
  * David McKnight    (IBM) 		 - [339548] [dstore] shouldn't attempt file conversion on empty files
  * David McKnight    (IBM)       - [365780] [dstore] codepage conversion should only occur for different encodings
+ * David McKnight   (IBM)        - [390037] [dstore] Duplicated items in the System view
  *******************************************************************************/
 
 package org.eclipse.rse.internal.services.dstore.files;
@@ -2112,7 +2113,8 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 		DataElement element = (DataElement)_fileElementMap.get(normalizedPath);
 		if (element != null)
 		{
-			if (element.isDeleted()){
+			if (element.isDeleted() 
+					|| ds.isDoSpirit()){ // when using spirit, don't use element cache 
 				_fileElementMap.remove(normalizedPath);
 				element = null;
 			}