[261644] [dstore] remote search improvements

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java

@@ -44,10 +44,12 @@ public class DataElementRemover extends Handler
 	private int _expiryTime = DEFAULT_EXPIRY_TIME * 10;
 	public static final String EXPIRY_TIME_PROPERTY_NAME = "SPIRIT_EXPIRY_TIME"; //$NON-NLS-1$
 	public static final String INTERVAL_TIME_PROPERTY_NAME = "SPIRIT_INTERVAL_TIME"; //$NON-NLS-1$
+	public MemoryManager _memoryManager;
 	
 	public DataElementRemover(DataStore dataStore)
 	{
 		super();
+		_memoryManager = MemoryManager.getInstance(dataStore);
 		_dataStore = dataStore;
 		_queue = new LinkedList();
 		getTimes();
@@ -120,16 +122,7 @@ public class DataElementRemover extends Handler
 	}
 
 	private boolean isMemoryThresholdExceeded(){
-		// trying to avoid using Java 1.5
+		return _memoryManager.isThresholdExceeded();
-		Runtime runtime = Runtime.getRuntime();
-		long freeMem = runtime.freeMemory();
-
-		if (freeMem < 10000){
-
-			return true;
-		}
-		
-		return false;
 	}
 	
 	public void handle()

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/MemoryManager.java

@@ -0,0 +1,156 @@
+/*******************************************************************************
+ * Copyright (c) 2009 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Initial Contributors:
+ * The following IBM employees contributed to the Remote System Explorer
+ * component that contains this file: David McKnight
+ * 
+ * Contributors:
+ * David McKnight  (IBM)  - [261644] [dstore] remote search improvements
+ ********************************************************************************/
+
+package org.eclipse.dstore.internal.core.util;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.List;
+
+import org.eclipse.dstore.core.model.DataStore;
+import org.eclipse.dstore.core.server.SystemServiceManager;
+
+public class MemoryManager {
+	private Object mbean;
+	private static MemoryManager _instance;
+	private DataStore _dataStore;
+	
+	private MemoryManager(DataStore dataStore) {
+		init();
+		_dataStore = dataStore;
+	}
+	
+	public static MemoryManager getInstance(DataStore dataStore){
+		if (_instance == null){
+			_instance = new MemoryManager(dataStore);
+		}
+		return _instance;
+	}
+	
+	private void init(){
+		String thresholdString = System.getProperty("search.threshold"); //$NON-NLS-1$
+		double threshold = 0.8;
+		if(thresholdString != null && thresholdString.length() > 0) {
+			threshold = (long) (Integer.parseInt(thresholdString) / 100);
+		}
+		
+		// do we have java 1.5?
+		try {
+			Class factoryClass = Class.forName("java.lang.management.ManagementFactory"); //$NON-NLS-1$
+			Method method = factoryClass.getDeclaredMethod("getMemoryPoolMXBeans", new Class[0]);  //$NON-NLS-1$
+			
+			List list = (List)method.invoke(null, null);
+			
+			for(int i = 0;i < list.size(); i++) {
+				Object mbObj = list.get(i);
+				Class mbClass = mbObj.getClass();
+				
+				Method getSupportedMethod = mbClass.getDeclaredMethod("isUsageThresholdSupported", new Class[0]);  //$NON-NLS-1$				
+
+				Boolean usageThresholdSupported = (Boolean)getSupportedMethod.invoke(mbObj, null);
+				if (usageThresholdSupported.booleanValue()){
+					
+					Method getTypeMethod = mbClass.getDeclaredMethod("getType", new Class[0]);  //$NON-NLS-1$
+				
+					Object typeObj = getTypeMethod.invoke(mbObj, null);
+					Class memoryType = Class.forName("java.lang.management.MemoryType");  //$NON-NLS-1$
+					Field field = memoryType.getField("HEAP");  //$NON-NLS-1$
+					Object fieldObj = field.get(typeObj);
+				
+					if (fieldObj.equals(typeObj)){						
+						Method getUsageMethod = mbClass.getDeclaredMethod("getUsage", new Class[0]);  //$NON-NLS-1$
+						Object usageObj = getUsageMethod.invoke(mbObj, null);
+						
+						Class usageClass = usageObj.getClass();						
+						Method getMaxMethod = usageClass.getDeclaredMethod("getMax", new Class[0]);  //$NON-NLS-1$
+						Long maxObj = (Long)getMaxMethod.invoke(usageObj, null);
+												
+						Method setThresholdMethod = mbClass.getDeclaredMethod("setUsageThreshold", new Class[] { long.class });  //$NON-NLS-1$
+						Object[] args = new Object[1];
+						args[0] = new Long((long)(maxObj.longValue() * threshold));
+						
+						setThresholdMethod.invoke(mbObj, args);
+						mbean = mbObj;
+						break;
+					}
+				}						
+			}
+		}
+		catch (Exception e){
+			// java version to old so no mbean created - will use fallback
+			e.printStackTrace();
+		}
+	}
+	
+	public boolean isThresholdExceeded() {
+					
+		if (mbean != null){
+			try {
+				Method method = mbean.getClass().getMethod("isUsageThresholdExceeded", new Class[0]);  //$NON-NLS-1$
+				Boolean exceeded = (Boolean)method.invoke(mbean, null);
+				return exceeded.booleanValue();
+			}
+			catch (Exception e){
+				return false;
+			}
+		}
+		else {						
+			// no Java 1.5 available, so this is the fallback
+			Runtime runtime = Runtime.getRuntime();
+			long freeMem = runtime.freeMemory();
+
+			if (freeMem < 10000){
+
+				return true;
+			}								
+			return false;
+		}
+	}
+	
+	public void checkAndClearupMemory()
+	{
+		int count = 0;
+		while(count < 5 && isThresholdExceeded()) {
+			System.gc();
+			try {
+				Thread.sleep(2000);
+			} catch (InterruptedException e) {
+			}
+			count ++;
+			_dataStore.trace("CLEAN free mem="+Runtime.getRuntime().freeMemory());  //$NON-NLS-1$
+		}
+		if(count == 5) { 
+
+			
+			_dataStore.trace("Out of memory - shutting down"); //$NON-NLS-1$
+			
+			Exception e = new Exception();
+			_dataStore.trace(e);
+
+			/*
+			 * show the end of the log
+			DataElement logRoot = _dataStore.getLogRoot();
+			List nestedData = logRoot.getNestedData();
+			for (int i = nestedData.size() - 10; i < nestedData.size(); i++){
+				DataElement cmd = (DataElement)nestedData.get(i);
+				System.out.println(cmd);
+			}
+			*/
+
+			if (SystemServiceManager.getInstance().getSystemService() == null)
+				System.exit(-1);
+		}				
+	}
+}

rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/internal/dstore/universal/miners/filesystem/UniversalSearchHandler.java

@@ -21,6 +21,7 @@
  * David McKnight  (IBM)  - [250168] handle malformed binary and always resolve canonical paths
  * David McKnight  (IBM)  - [250168] update to just search file of canonical paths (not symbolic links)
  * David McKnight  (IBM)  - [255390] memory checking
+ * David McKnight  (IBM)  - [261644] [dstore] remote search improvements
  ********************************************************************************/
 
 package org.eclipse.rse.internal.dstore.universal.miners.filesystem;
@@ -29,7 +30,10 @@ import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStreamReader;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 import java.util.HashSet;
+import java.util.List;
 
 import org.eclipse.dstore.core.model.DE;
 import org.eclipse.dstore.core.model.DataElement;
@@ -37,6 +41,7 @@ import org.eclipse.dstore.core.model.DataStore;
 import org.eclipse.dstore.core.server.SecuredThread;
 import org.eclipse.dstore.core.server.SystemServiceManager;
 import org.eclipse.dstore.core.util.StringCompare;
+import org.eclipse.dstore.internal.core.util.MemoryManager;
 import org.eclipse.rse.dstore.universal.miners.ICancellableHandler;
 import org.eclipse.rse.dstore.universal.miners.IUniversalDataStoreConstants;
 import org.eclipse.rse.dstore.universal.miners.UniversalFileSystemMiner;
@@ -78,9 +83,12 @@ public class UniversalSearchHandler extends SecuredThread implements ICancellabl
 	protected DataElement _deVirtualFile;
 
 	protected boolean _fsCaseSensitive;
+	private MemoryManager _memoryManager;
 
 	public UniversalSearchHandler(DataStore dataStore, UniversalFileSystemMiner miner, SystemSearchString searchString, boolean fsCaseSensitive, File theFile, DataElement status) {
 		super(dataStore);
+		
+		_memoryManager = MemoryManager.getInstance(dataStore);
 		_miner = miner;
 		_searchString = searchString;
 		_fsCaseSensitive = fsCaseSensitive;
@@ -279,7 +287,8 @@ public class UniversalSearchHandler extends SecuredThread implements ICancellabl
 				}
 	
 				// do a refresh
-				//_dataStore.refresh(_status);
+				_dataStore.refresh(_status);
+				_dataStore.disconnectObjects(_status);
 			}
 	
 			// if the depth is not 0, then we need to recursively search
@@ -351,15 +360,25 @@ public class UniversalSearchHandler extends SecuredThread implements ICancellabl
 		FileInputStream inputStream = null;
 
 		try {
+			long MAX_FILE = Runtime.getRuntime().freeMemory() / 4;
+			long fileLength = theFile.length();
+			
+			//System.out.println("file="+absPath);
+			//System.out.println("MAX mem="+MAX_FILE);
+			//System.out.println("fileLength="+fileLength);
+			//if (fileLength < MAX_FILE)
+			if (true){ // if the file is too big, the server will run out of memory
 				inputStream = new FileInputStream(theFile);
 				InputStreamReader reader = new InputStreamReader(inputStream);		
 				BufferedReader bufReader = new BufferedReader(reader);
 	
 				// test for unreadable binary
-			if (isUnreadableBinary(bufReader)){
+				if (isUnreadableBinary(bufReader) || fileLength > MAX_FILE){
 					// search some other way?
 					long size = theFile.length();
 					if (simpleSearch(inputStream, size, _stringMatcher)){			
+						bufReader.close();
+						reader.close();
 						return true;
 					}
 					
@@ -367,21 +386,26 @@ public class UniversalSearchHandler extends SecuredThread implements ICancellabl
 					reader.close();
 					return false;
 				}
-			else {
+				else 
+				{
 					SystemSearchStringMatchLocator locator = new SystemSearchStringMatchLocator(bufReader, _stringMatcher);						
 					
 					SystemSearchLineMatch[] matches = locator.locateMatches();									
 					boolean foundMatches = ((matches != null) && (matches.length > 0));
 	
 					if (foundMatches) {
+						if (matches.length * 500 < MAX_FILE){ // only creating match objects if we have enough memory																			
 							convert(remoteFile, absPath, matches);
 						}
-
+					}
-				bufReader.close();
-				reader.close();
 					return foundMatches;
 				} 
 			}
+			else {
+				_dataStore.trace("skipped:"+absPath  + " due to memory constraints"); //$NON-NLS-1$ //$NON-NLS-2$
+				return false;
+			}
+		}
 		catch (OutOfMemoryError e){
 			if (SystemServiceManager.getInstance().getSystemService() == null)
 				System.exit(-1);
@@ -417,6 +441,7 @@ public class UniversalSearchHandler extends SecuredThread implements ICancellabl
 		catch (Exception e){
 			return true;
 		}
+		
 		return false;
 	}
 
@@ -454,38 +479,16 @@ public class UniversalSearchHandler extends SecuredThread implements ICancellabl
 			match = lineMatches[i];
 			DataElement obj = _dataStore.createObject(deObj, _deGrep, match.getLine(), absPath);
 			obj.setAttribute(DE.A_SOURCE, obj.getSource() + ':'+ match.getLineNumber());
+	
 		}
+		_dataStore.disconnectObjects(deObj);	
 	}
 	
 	public void checkAndClearupMemory()
 	{
-		int count = 0;
+		_memoryManager.checkAndClearupMemory();			
-		while(count < 5 && isMemoryThresholdExceeded()) {
-			
-			System.gc();
-			try {
-				Thread.sleep(2000);
-			} catch (InterruptedException e) {
-			}
-			count ++;
-		}
-		if(count == 5) { 
-			_dataStore.trace("heap memory low"); //$NON-NLS-1$
-			if (SystemServiceManager.getInstance().getSystemService() == null)
-				System.exit(-1);
-		}				
 	}
 
-	private boolean isMemoryThresholdExceeded(){
-		// trying to avoid using Java 1.5
-		Runtime runtime = Runtime.getRuntime();
-		long freeMem = runtime.freeMemory();
 	
-		if (freeMem < 10000){
 		
-			return true;
-		}
-		
-		return false;
-	}
 }

rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/search/DStoreSearchService.java

@@ -19,6 +19,7 @@
  * David McKnight   (IBM)        - [214378] don't mark as finished until we have the results - sleep instead of wait
  * David McKnight   (IBM)        - [216252] use SimpleSystemMessage instead of getMessage()
  * David McKnight  (IBM)  - [255390] don't assume one update means the search is done
+ * David McKnight  (IBM)  - [261644] [dstore] remote search improvements
  *******************************************************************************/
 
 package org.eclipse.rse.internal.services.dstore.search;
@@ -94,7 +95,9 @@ public class DStoreSearchService extends AbstractDStoreService implements ISearc
 			{
 				boolean working = true;
 				while (working){
-					getStatusMonitor(ds).waitForUpdate(status, monitor);
+					// give large wait time for a search
+					int waitThres = -1;
+					getStatusMonitor(ds).waitForUpdate(status, monitor, waitThres);
 					String statusStr = status.getName();
 					if (statusStr.equals("done")) //$NON-NLS-1$
 					{

rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/util/DStoreStatusMonitor.java

@@ -18,6 +18,7 @@
  * David McKnight   (IBM)        - [209593] [api] check for existing query to avoid duplicates
  * David McKnight   (IBM)        - [225902] [dstore] use C_NOTIFICATION command to wake up the server
  * David McKnight   (IBM)        - [231126] [dstore] status monitor needs to reset WaitThreshold on nudge
+ * David McKnight  (IBM)  - [261644] [dstore] remote search improvements
  *******************************************************************************/
 
 package org.eclipse.rse.services.dstore.util;
@@ -226,12 +227,12 @@ public class DStoreStatusMonitor implements IDomainListener
 
 	 public DataElement waitForUpdate(DataElement status) throws InterruptedException
 		{
-	        return waitForUpdate(status, null, 0);
+	        return waitForUpdate(status, null, 1000);
 		}
 
     public DataElement waitForUpdate(DataElement status, IProgressMonitor monitor) throws InterruptedException
 	{
-        return waitForUpdate(status, monitor, 0);
+        return waitForUpdate(status, monitor, 1000);
 	}
 
     public DataElement waitForUpdate(DataElement status, int wait) throws InterruptedException
@@ -284,6 +285,10 @@ public class DStoreStatusMonitor implements IDomainListener
 
 					waitForUpdate();
                     //Thread.sleep(200);
+					if (!status.getDataStore().isConnected()){
+						// not connected anymore!
+						_networkDown = true;
+					}
 
                     if (WaitThreshold > 0) // update timer count if
                         // threshold not reached

rse/plugins/org.eclipse.rse.subsystems.files.dstore/src/org/eclipse/rse/internal/subsystems/files/dstore/DStoreFileSubSystemSearchResultConfiguration.java

@@ -18,6 +18,7 @@
  * David McKnight   (IBM)        - [190010] performance improvement to use caching for dstore search
  * David McKnight   (IBM)        - [207178] changing list APIs for file service and subsystems
  * David McKnight   (IBM)        - [214378] [dstore] remote search doesn't display results sometimes
+ * David McKnight  (IBM)  - [261644] [dstore] remote search improvements
  *******************************************************************************/
 
 package org.eclipse.rse.internal.subsystems.files.dstore;
@@ -26,14 +27,13 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 
-import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.dstore.core.model.DataElement;
 import org.eclipse.dstore.extra.DomainEvent;
-import org.eclipse.rse.core.subsystems.RemoteChildrenContentsType;
+import org.eclipse.rse.internal.services.dstore.files.DStoreHostFile;
 import org.eclipse.rse.internal.services.dstore.search.DStoreSearchResultConfiguration;
 import org.eclipse.rse.services.clientserver.SystemSearchString;
-import org.eclipse.rse.services.clientserver.messages.SystemMessageException;
+import org.eclipse.rse.services.files.IHostFile;
 import org.eclipse.rse.services.search.IHostSearchConstants;
 import org.eclipse.rse.services.search.IHostSearchResult;
 import org.eclipse.rse.services.search.IHostSearchResultSet;
@@ -42,15 +42,17 @@ import org.eclipse.rse.subsystems.files.core.servicesubsystem.FileServiceSubSyst
 import org.eclipse.rse.subsystems.files.core.servicesubsystem.OutputRefresh;
 import org.eclipse.rse.subsystems.files.core.subsystems.IHostFileToRemoteFileAdapter;
 import org.eclipse.rse.subsystems.files.core.subsystems.IRemoteFile;
+import org.eclipse.rse.subsystems.files.core.subsystems.IRemoteFileContext;
+import org.eclipse.rse.subsystems.files.core.subsystems.RemoteFileContext;
 import org.eclipse.rse.subsystems.files.core.subsystems.RemoteSearchResultsContentsType;
-import org.eclipse.rse.ui.messages.SystemMessageDialog;
 import org.eclipse.swt.widgets.Display;
 
 public class DStoreFileSubSystemSearchResultConfiguration extends DStoreSearchResultConfiguration
 {
+
+	
 	private FileServiceSubSystem _fileSubSystem;
 	private IRemoteFile _searchObject;
-	private HashMap _parentCache;
 	private List    _convertedResults;
 
 	public DStoreFileSubSystemSearchResultConfiguration(IHostSearchResultSet set, Object searchObject, SystemSearchString searchString, ISearchService searchService, IHostFileToRemoteFileAdapter fileAdapter)
@@ -58,7 +60,6 @@ public class DStoreFileSubSystemSearchResultConfiguration extends DStoreSearchRe
 		super(set, searchObject, searchString, searchService);
 		_searchObject = (IRemoteFile)searchObject;
 		_fileSubSystem = (FileServiceSubSystem)_searchObject.getParentRemoteFileSubSystem();
-		_parentCache = new HashMap();
 		_convertedResults = new ArrayList();
 	}
 
@@ -85,37 +86,16 @@ public class DStoreFileSubSystemSearchResultConfiguration extends DStoreSearchRe
 		{
 			if (results.size() > _convertedResults.size())
 			{
-			IProgressMonitor monitor = new NullProgressMonitor();	
+			for (int i = _convertedResults.size(); i < results.size(); i++)
-			for (int i = 0; i < results.size(); i++)
 			{ 
 				DataElement fileNode = (DataElement)results.get(i);
 				if (fileNode != null && !fileNode.getType().equals("error")) //$NON-NLS-1$
 				{
-					IRemoteFile parentRemoteFile = null;
 					try
 					{
-						parentRemoteFile = (IRemoteFile)_parentCache.get(fileNode.getValue());
+						IHostFile hostFile = new DStoreHostFile(fileNode);
-						if (parentRemoteFile == null)
+						IRemoteFileContext context = _fileSubSystem.getTheDefaultContext();
-						{
+						IRemoteFile remoteFile = _fileSubSystem.getHostFileToRemoteFileAdapter().convertToRemoteFile(_fileSubSystem, context, null, hostFile);
-							parentRemoteFile = _fileSubSystem.getRemoteFileObject(fileNode.getValue(),monitor);
-							_parentCache.put(fileNode.getValue(), parentRemoteFile);
-		
-						
-							if (parentRemoteFile != null && !parentRemoteFile.hasContents(RemoteChildrenContentsType.getInstance()))
-							{
-								// query all files to save time (so we can retrieve cached files
-								IRemoteFile[] children = _fileSubSystem.list(parentRemoteFile, monitor);
-								for (int c = 0; c < children.length; c++)
-								{
-									if (!children[c].isFile())
-									{
-										_parentCache.put(children[c].getAbsolutePath(), children[c]);
-									}
-								}
-							}
-						}
-						String path = fileNode.getValue() + "/" + fileNode.getName(); //$NON-NLS-1$
-						IRemoteFile remoteFile = _fileSubSystem.getRemoteFileObject(path, monitor);
 
 						List contained = fileNode.getNestedData();
 						if (contained != null)
@@ -130,13 +110,8 @@ public class DStoreFileSubSystemSearchResultConfiguration extends DStoreSearchRe
 						}
 						_convertedResults.add(remoteFile);
 					}
-					catch (SystemMessageException e)
-					{
-						SystemMessageDialog.displayMessage(e);
-					}
 					catch (Exception e)
 					{						
-						
 					}
 				}
 			}