diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java index 529dacdd28a..615650ea157 100644 --- a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java @@ -44,10 +44,12 @@ public class DataElementRemover extends Handler private int _expiryTime = DEFAULT_EXPIRY_TIME * 10; public static final String EXPIRY_TIME_PROPERTY_NAME = "SPIRIT_EXPIRY_TIME"; //$NON-NLS-1$ public static final String INTERVAL_TIME_PROPERTY_NAME = "SPIRIT_INTERVAL_TIME"; //$NON-NLS-1$ + public MemoryManager _memoryManager; public DataElementRemover(DataStore dataStore) { super(); + _memoryManager = MemoryManager.getInstance(dataStore); _dataStore = dataStore; _queue = new LinkedList(); getTimes(); @@ -120,16 +122,7 @@ public class DataElementRemover extends Handler } private boolean isMemoryThresholdExceeded(){ - // trying to avoid using Java 1.5 - Runtime runtime = Runtime.getRuntime(); - long freeMem = runtime.freeMemory(); - - if (freeMem < 10000){ - - return true; - } - - return false; + return _memoryManager.isThresholdExceeded(); } public void handle() diff --git a/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/MemoryManager.java b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/MemoryManager.java new file mode 100644 index 00000000000..9fbb43b8b53 --- /dev/null +++ b/rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/MemoryManager.java @@ -0,0 +1,156 @@ +/******************************************************************************* + * Copyright (c) 2009 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Initial Contributors: + * The following IBM employees contributed to the Remote System Explorer + * component that contains this file: David McKnight + * + * Contributors: + * David McKnight (IBM) - [261644] [dstore] remote search improvements + ********************************************************************************/ + +package org.eclipse.dstore.internal.core.util; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.List; + +import org.eclipse.dstore.core.model.DataStore; +import org.eclipse.dstore.core.server.SystemServiceManager; + +public class MemoryManager { + private Object mbean; + private static MemoryManager _instance; + private DataStore _dataStore; + + private MemoryManager(DataStore dataStore) { + init(); + _dataStore = dataStore; + } + + public static MemoryManager getInstance(DataStore dataStore){ + if (_instance == null){ + _instance = new MemoryManager(dataStore); + } + return _instance; + } + + private void init(){ + String thresholdString = System.getProperty("search.threshold"); //$NON-NLS-1$ + double threshold = 0.8; + if(thresholdString != null && thresholdString.length() > 0) { + threshold = (long) (Integer.parseInt(thresholdString) / 100); + } + + // do we have java 1.5? + try { + Class factoryClass = Class.forName("java.lang.management.ManagementFactory"); //$NON-NLS-1$ + Method method = factoryClass.getDeclaredMethod("getMemoryPoolMXBeans", new Class[0]); //$NON-NLS-1$ + + List list = (List)method.invoke(null, null); + + for(int i = 0;i < list.size(); i++) { + Object mbObj = list.get(i); + Class mbClass = mbObj.getClass(); + + Method getSupportedMethod = mbClass.getDeclaredMethod("isUsageThresholdSupported", new Class[0]); //$NON-NLS-1$ + + Boolean usageThresholdSupported = (Boolean)getSupportedMethod.invoke(mbObj, null); + if (usageThresholdSupported.booleanValue()){ + + Method getTypeMethod = mbClass.getDeclaredMethod("getType", new Class[0]); //$NON-NLS-1$ + + Object typeObj = getTypeMethod.invoke(mbObj, null); + Class memoryType = Class.forName("java.lang.management.MemoryType"); //$NON-NLS-1$ + Field field = memoryType.getField("HEAP"); //$NON-NLS-1$ + Object fieldObj = field.get(typeObj); + + if (fieldObj.equals(typeObj)){ + Method getUsageMethod = mbClass.getDeclaredMethod("getUsage", new Class[0]); //$NON-NLS-1$ + Object usageObj = getUsageMethod.invoke(mbObj, null); + + Class usageClass = usageObj.getClass(); + Method getMaxMethod = usageClass.getDeclaredMethod("getMax", new Class[0]); //$NON-NLS-1$ + Long maxObj = (Long)getMaxMethod.invoke(usageObj, null); + + Method setThresholdMethod = mbClass.getDeclaredMethod("setUsageThreshold", new Class[] { long.class }); //$NON-NLS-1$ + Object[] args = new Object[1]; + args[0] = new Long((long)(maxObj.longValue() * threshold)); + + setThresholdMethod.invoke(mbObj, args); + mbean = mbObj; + break; + } + } + } + } + catch (Exception e){ + // java version to old so no mbean created - will use fallback + e.printStackTrace(); + } + } + + public boolean isThresholdExceeded() { + + if (mbean != null){ + try { + Method method = mbean.getClass().getMethod("isUsageThresholdExceeded", new Class[0]); //$NON-NLS-1$ + Boolean exceeded = (Boolean)method.invoke(mbean, null); + return exceeded.booleanValue(); + } + catch (Exception e){ + return false; + } + } + else { + // no Java 1.5 available, so this is the fallback + Runtime runtime = Runtime.getRuntime(); + long freeMem = runtime.freeMemory(); + + if (freeMem < 10000){ + + return true; + } + return false; + } + } + + public void checkAndClearupMemory() + { + int count = 0; + while(count < 5 && isThresholdExceeded()) { + System.gc(); + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + } + count ++; + _dataStore.trace("CLEAN free mem="+Runtime.getRuntime().freeMemory()); //$NON-NLS-1$ + } + if(count == 5) { + + + _dataStore.trace("Out of memory - shutting down"); //$NON-NLS-1$ + + Exception e = new Exception(); + _dataStore.trace(e); + + /* + * show the end of the log + DataElement logRoot = _dataStore.getLogRoot(); + List nestedData = logRoot.getNestedData(); + for (int i = nestedData.size() - 10; i < nestedData.size(); i++){ + DataElement cmd = (DataElement)nestedData.get(i); + System.out.println(cmd); + } + */ + + if (SystemServiceManager.getInstance().getSystemService() == null) + System.exit(-1); + } + } +} diff --git a/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/internal/dstore/universal/miners/filesystem/UniversalSearchHandler.java b/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/internal/dstore/universal/miners/filesystem/UniversalSearchHandler.java index 191a653d79a..164103e04b8 100644 --- a/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/internal/dstore/universal/miners/filesystem/UniversalSearchHandler.java +++ b/rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/internal/dstore/universal/miners/filesystem/UniversalSearchHandler.java @@ -21,6 +21,7 @@ * David McKnight (IBM) - [250168] handle malformed binary and always resolve canonical paths * David McKnight (IBM) - [250168] update to just search file of canonical paths (not symbolic links) * David McKnight (IBM) - [255390] memory checking + * David McKnight (IBM) - [261644] [dstore] remote search improvements ********************************************************************************/ package org.eclipse.rse.internal.dstore.universal.miners.filesystem; @@ -29,7 +30,10 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.InputStreamReader; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.HashSet; +import java.util.List; import org.eclipse.dstore.core.model.DE; import org.eclipse.dstore.core.model.DataElement; @@ -37,6 +41,7 @@ import org.eclipse.dstore.core.model.DataStore; import org.eclipse.dstore.core.server.SecuredThread; import org.eclipse.dstore.core.server.SystemServiceManager; import org.eclipse.dstore.core.util.StringCompare; +import org.eclipse.dstore.internal.core.util.MemoryManager; import org.eclipse.rse.dstore.universal.miners.ICancellableHandler; import org.eclipse.rse.dstore.universal.miners.IUniversalDataStoreConstants; import org.eclipse.rse.dstore.universal.miners.UniversalFileSystemMiner; @@ -78,9 +83,12 @@ public class UniversalSearchHandler extends SecuredThread implements ICancellabl protected DataElement _deVirtualFile; protected boolean _fsCaseSensitive; + private MemoryManager _memoryManager; public UniversalSearchHandler(DataStore dataStore, UniversalFileSystemMiner miner, SystemSearchString searchString, boolean fsCaseSensitive, File theFile, DataElement status) { super(dataStore); + + _memoryManager = MemoryManager.getInstance(dataStore); _miner = miner; _searchString = searchString; _fsCaseSensitive = fsCaseSensitive; @@ -279,7 +287,8 @@ public class UniversalSearchHandler extends SecuredThread implements ICancellabl } // do a refresh - //_dataStore.refresh(_status); + _dataStore.refresh(_status); + _dataStore.disconnectObjects(_status); } // if the depth is not 0, then we need to recursively search @@ -351,35 +360,50 @@ public class UniversalSearchHandler extends SecuredThread implements ICancellabl FileInputStream inputStream = null; try { - inputStream = new FileInputStream(theFile); - InputStreamReader reader = new InputStreamReader(inputStream); - BufferedReader bufReader = new BufferedReader(reader); + long MAX_FILE = Runtime.getRuntime().freeMemory() / 4; + long fileLength = theFile.length(); - // test for unreadable binary - if (isUnreadableBinary(bufReader)){ - // search some other way? - long size = theFile.length(); - if (simpleSearch(inputStream, size, _stringMatcher)){ - return true; + //System.out.println("file="+absPath); + //System.out.println("MAX mem="+MAX_FILE); + //System.out.println("fileLength="+fileLength); + //if (fileLength < MAX_FILE) + if (true){ // if the file is too big, the server will run out of memory + inputStream = new FileInputStream(theFile); + InputStreamReader reader = new InputStreamReader(inputStream); + BufferedReader bufReader = new BufferedReader(reader); + + // test for unreadable binary + if (isUnreadableBinary(bufReader) || fileLength > MAX_FILE){ + // search some other way? + long size = theFile.length(); + if (simpleSearch(inputStream, size, _stringMatcher)){ + bufReader.close(); + reader.close(); + return true; + } + + bufReader.close(); + reader.close(); + return false; } - - bufReader.close(); - reader.close(); - return false; + else + { + SystemSearchStringMatchLocator locator = new SystemSearchStringMatchLocator(bufReader, _stringMatcher); + + SystemSearchLineMatch[] matches = locator.locateMatches(); + boolean foundMatches = ((matches != null) && (matches.length > 0)); + + if (foundMatches) { + if (matches.length * 500 < MAX_FILE){ // only creating match objects if we have enough memory + convert(remoteFile, absPath, matches); + } + } + return foundMatches; + } } else { - SystemSearchStringMatchLocator locator = new SystemSearchStringMatchLocator(bufReader, _stringMatcher); - - SystemSearchLineMatch[] matches = locator.locateMatches(); - boolean foundMatches = ((matches != null) && (matches.length > 0)); - - if (foundMatches) { - convert(remoteFile, absPath, matches); - } - - bufReader.close(); - reader.close(); - return foundMatches; + _dataStore.trace("skipped:"+absPath + " due to memory constraints"); //$NON-NLS-1$ //$NON-NLS-2$ + return false; } } catch (OutOfMemoryError e){ @@ -417,6 +441,7 @@ public class UniversalSearchHandler extends SecuredThread implements ICancellabl catch (Exception e){ return true; } + return false; } @@ -449,43 +474,21 @@ public class UniversalSearchHandler extends SecuredThread implements ICancellabl protected void convert(DataElement deObj, String absPath, SystemSearchLineMatch[] lineMatches) { SystemSearchLineMatch match = null; - + for (int i = 0; i < lineMatches.length; i++) { match = lineMatches[i]; DataElement obj = _dataStore.createObject(deObj, _deGrep, match.getLine(), absPath); obj.setAttribute(DE.A_SOURCE, obj.getSource() + ':'+ match.getLineNumber()); + } + _dataStore.disconnectObjects(deObj); } public void checkAndClearupMemory() { - int count = 0; - while(count < 5 && isMemoryThresholdExceeded()) { - - System.gc(); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - } - count ++; - } - if(count == 5) { - _dataStore.trace("heap memory low"); //$NON-NLS-1$ - if (SystemServiceManager.getInstance().getSystemService() == null) - System.exit(-1); - } + _memoryManager.checkAndClearupMemory(); } - private boolean isMemoryThresholdExceeded(){ - // trying to avoid using Java 1.5 - Runtime runtime = Runtime.getRuntime(); - long freeMem = runtime.freeMemory(); - - if (freeMem < 10000){ - - return true; - } + - return false; - } } diff --git a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/search/DStoreSearchService.java b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/search/DStoreSearchService.java index 10255bbc47f..0f06c384f08 100644 --- a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/search/DStoreSearchService.java +++ b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/search/DStoreSearchService.java @@ -19,6 +19,7 @@ * David McKnight (IBM) - [214378] don't mark as finished until we have the results - sleep instead of wait * David McKnight (IBM) - [216252] use SimpleSystemMessage instead of getMessage() * David McKnight (IBM) - [255390] don't assume one update means the search is done + * David McKnight (IBM) - [261644] [dstore] remote search improvements *******************************************************************************/ package org.eclipse.rse.internal.services.dstore.search; @@ -94,7 +95,9 @@ public class DStoreSearchService extends AbstractDStoreService implements ISearc { boolean working = true; while (working){ - getStatusMonitor(ds).waitForUpdate(status, monitor); + // give large wait time for a search + int waitThres = -1; + getStatusMonitor(ds).waitForUpdate(status, monitor, waitThres); String statusStr = status.getName(); if (statusStr.equals("done")) //$NON-NLS-1$ { diff --git a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/util/DStoreStatusMonitor.java b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/util/DStoreStatusMonitor.java index 8ef2e9668ed..3e12ffba25d 100644 --- a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/util/DStoreStatusMonitor.java +++ b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/util/DStoreStatusMonitor.java @@ -18,6 +18,7 @@ * David McKnight (IBM) - [209593] [api] check for existing query to avoid duplicates * David McKnight (IBM) - [225902] [dstore] use C_NOTIFICATION command to wake up the server * David McKnight (IBM) - [231126] [dstore] status monitor needs to reset WaitThreshold on nudge + * David McKnight (IBM) - [261644] [dstore] remote search improvements *******************************************************************************/ package org.eclipse.rse.services.dstore.util; @@ -226,12 +227,12 @@ public class DStoreStatusMonitor implements IDomainListener public DataElement waitForUpdate(DataElement status) throws InterruptedException { - return waitForUpdate(status, null, 0); + return waitForUpdate(status, null, 1000); } public DataElement waitForUpdate(DataElement status, IProgressMonitor monitor) throws InterruptedException { - return waitForUpdate(status, monitor, 0); + return waitForUpdate(status, monitor, 1000); } public DataElement waitForUpdate(DataElement status, int wait) throws InterruptedException @@ -284,6 +285,10 @@ public class DStoreStatusMonitor implements IDomainListener waitForUpdate(); //Thread.sleep(200); + if (!status.getDataStore().isConnected()){ + // not connected anymore! + _networkDown = true; + } if (WaitThreshold > 0) // update timer count if // threshold not reached diff --git a/rse/plugins/org.eclipse.rse.subsystems.files.dstore/src/org/eclipse/rse/internal/subsystems/files/dstore/DStoreFileSubSystemSearchResultConfiguration.java b/rse/plugins/org.eclipse.rse.subsystems.files.dstore/src/org/eclipse/rse/internal/subsystems/files/dstore/DStoreFileSubSystemSearchResultConfiguration.java index f66728cdba2..bec4ae1a729 100644 --- a/rse/plugins/org.eclipse.rse.subsystems.files.dstore/src/org/eclipse/rse/internal/subsystems/files/dstore/DStoreFileSubSystemSearchResultConfiguration.java +++ b/rse/plugins/org.eclipse.rse.subsystems.files.dstore/src/org/eclipse/rse/internal/subsystems/files/dstore/DStoreFileSubSystemSearchResultConfiguration.java @@ -18,6 +18,7 @@ * David McKnight (IBM) - [190010] performance improvement to use caching for dstore search * David McKnight (IBM) - [207178] changing list APIs for file service and subsystems * David McKnight (IBM) - [214378] [dstore] remote search doesn't display results sometimes + * David McKnight (IBM) - [261644] [dstore] remote search improvements *******************************************************************************/ package org.eclipse.rse.internal.subsystems.files.dstore; @@ -26,14 +27,13 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.core.runtime.NullProgressMonitor; import org.eclipse.dstore.core.model.DataElement; import org.eclipse.dstore.extra.DomainEvent; -import org.eclipse.rse.core.subsystems.RemoteChildrenContentsType; +import org.eclipse.rse.internal.services.dstore.files.DStoreHostFile; import org.eclipse.rse.internal.services.dstore.search.DStoreSearchResultConfiguration; import org.eclipse.rse.services.clientserver.SystemSearchString; -import org.eclipse.rse.services.clientserver.messages.SystemMessageException; +import org.eclipse.rse.services.files.IHostFile; import org.eclipse.rse.services.search.IHostSearchConstants; import org.eclipse.rse.services.search.IHostSearchResult; import org.eclipse.rse.services.search.IHostSearchResultSet; @@ -42,15 +42,17 @@ import org.eclipse.rse.subsystems.files.core.servicesubsystem.FileServiceSubSyst import org.eclipse.rse.subsystems.files.core.servicesubsystem.OutputRefresh; import org.eclipse.rse.subsystems.files.core.subsystems.IHostFileToRemoteFileAdapter; import org.eclipse.rse.subsystems.files.core.subsystems.IRemoteFile; +import org.eclipse.rse.subsystems.files.core.subsystems.IRemoteFileContext; +import org.eclipse.rse.subsystems.files.core.subsystems.RemoteFileContext; import org.eclipse.rse.subsystems.files.core.subsystems.RemoteSearchResultsContentsType; -import org.eclipse.rse.ui.messages.SystemMessageDialog; import org.eclipse.swt.widgets.Display; public class DStoreFileSubSystemSearchResultConfiguration extends DStoreSearchResultConfiguration { + + private FileServiceSubSystem _fileSubSystem; private IRemoteFile _searchObject; - private HashMap _parentCache; private List _convertedResults; public DStoreFileSubSystemSearchResultConfiguration(IHostSearchResultSet set, Object searchObject, SystemSearchString searchString, ISearchService searchService, IHostFileToRemoteFileAdapter fileAdapter) @@ -58,7 +60,6 @@ public class DStoreFileSubSystemSearchResultConfiguration extends DStoreSearchRe super(set, searchObject, searchString, searchService); _searchObject = (IRemoteFile)searchObject; _fileSubSystem = (FileServiceSubSystem)_searchObject.getParentRemoteFileSubSystem(); - _parentCache = new HashMap(); _convertedResults = new ArrayList(); } @@ -85,38 +86,17 @@ public class DStoreFileSubSystemSearchResultConfiguration extends DStoreSearchRe { if (results.size() > _convertedResults.size()) { - IProgressMonitor monitor = new NullProgressMonitor(); - for (int i = 0; i < results.size(); i++) - { + for (int i = _convertedResults.size(); i < results.size(); i++) + { DataElement fileNode = (DataElement)results.get(i); if (fileNode != null && !fileNode.getType().equals("error")) //$NON-NLS-1$ { - IRemoteFile parentRemoteFile = null; try { - parentRemoteFile = (IRemoteFile)_parentCache.get(fileNode.getValue()); - if (parentRemoteFile == null) - { - parentRemoteFile = _fileSubSystem.getRemoteFileObject(fileNode.getValue(),monitor); - _parentCache.put(fileNode.getValue(), parentRemoteFile); - - - if (parentRemoteFile != null && !parentRemoteFile.hasContents(RemoteChildrenContentsType.getInstance())) - { - // query all files to save time (so we can retrieve cached files - IRemoteFile[] children = _fileSubSystem.list(parentRemoteFile, monitor); - for (int c = 0; c < children.length; c++) - { - if (!children[c].isFile()) - { - _parentCache.put(children[c].getAbsolutePath(), children[c]); - } - } - } - } - String path = fileNode.getValue() + "/" + fileNode.getName(); //$NON-NLS-1$ - IRemoteFile remoteFile = _fileSubSystem.getRemoteFileObject(path, monitor); - + IHostFile hostFile = new DStoreHostFile(fileNode); + IRemoteFileContext context = _fileSubSystem.getTheDefaultContext(); + IRemoteFile remoteFile = _fileSubSystem.getHostFileToRemoteFileAdapter().convertToRemoteFile(_fileSubSystem, context, null, hostFile); + List contained = fileNode.getNestedData(); if (contained != null) { @@ -130,13 +110,8 @@ public class DStoreFileSubSystemSearchResultConfiguration extends DStoreSearchRe } _convertedResults.add(remoteFile); } - catch (SystemMessageException e) - { - SystemMessageDialog.displayMessage(e); - } catch (Exception e) - { - + { } } }