From 2836aac7aac0b49875c422a86ab2f356f62375ad Mon Sep 17 00:00:00 2001 From: David McKnight Date: Thu, 1 Nov 2007 19:32:30 +0000 Subject: [PATCH] [207178] optimizations for listMulti and getFileMulti for DstoreFlieService --- .../dstore/files/DStoreFileService.java | 115 +++++++++++---- .../dstore/AbstractDStoreService.java | 139 +++++++++++++----- .../dstore/util/DStoreStatusMonitor.java | 26 +++- 3 files changed, 216 insertions(+), 64 deletions(-) diff --git a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java index b2262723fb9..8801b3bd0d0 100644 --- a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java +++ b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java @@ -784,7 +784,7 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer return true; } - public IHostFile getFile(String remoteParent, String name, IProgressMonitor monitor) + private DataElement getSubjectFor(String remoteParent, String name) { DataElement de = null; if (name.equals(".") && name.equals(remoteParent)) //$NON-NLS-1$ @@ -801,10 +801,27 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer buf.append(name); de = getElementFor(buf.toString()); } + return de; + } + + private DataElement[] getSubjectsFor(String[] remoteParents, String[] names) + { + List subjects = new ArrayList(); + for (int i = 0; i < remoteParents.length; i++) + { + DataElement de = getSubjectFor(remoteParents[i], names[i]); + subjects.add(de); + } + return (DataElement[])subjects.toArray(new DataElement[subjects.size()]); + } + + public IHostFile getFile(String remoteParent, String name, IProgressMonitor monitor) + { + DataElement de = getSubjectFor(remoteParent, name); // with 207095, it's possible to get here unconnected such that there is no element if (de != null) { - dsQueryCommand(de, IUniversalDataStoreConstants.C_QUERY_GET_REMOTE_OBJECT, monitor); + dsQueryCommand(de, null, IUniversalDataStoreConstants.C_QUERY_GET_REMOTE_OBJECT, monitor); //getFile call should also need to convert this DataElement into a HostFile using //convertToHostFile() call. This way, this DataElement will be put into _fileMap. return convertToHostFile(de); @@ -814,16 +831,17 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer } } + /** + * Mass query of individual files + */ public IHostFile[] getFileMulti(String remoteParents[], String names[], IProgressMonitor monitor) throws SystemMessageException { - // TODO optimize dstore version of this to do mass queries then wait for last status - List results = new ArrayList(); - for (int i = 0; i < remoteParents.length; i++) - { - results.add(getFile(remoteParents[i], names[i], monitor)); - } - return (IHostFile[])results.toArray(new IHostFile[results.size()]); + DataElement[] subjects = getSubjectsFor(remoteParents, names); + + dsQueryCommandMulti(subjects, null, IUniversalDataStoreConstants.C_QUERY_GET_REMOTE_OBJECT, monitor); + + return convertToHostFiles(subjects, "*"); //$NON-NLS-1$ } /** @@ -1303,9 +1321,7 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer return convertToHostFiles(results, "*"); //$NON-NLS-1$ } - - - public IHostFile[] list(String remoteParent, String fileFilter, int fileType, IProgressMonitor monitor) + private String getQueryString(int fileType) { String queryString = null; switch (fileType) @@ -1323,7 +1339,12 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer queryString = IUniversalDataStoreConstants.C_QUERY_VIEW_ALL; break; } - + return queryString; + } + + public IHostFile[] list(String remoteParent, String fileFilter, int fileType, IProgressMonitor monitor) + { + String queryString = getQueryString(fileType); return fetch(remoteParent, fileFilter, queryString, monitor); } @@ -1332,20 +1353,9 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer String[] fileFilters, int fileType, IProgressMonitor monitor) throws SystemMessageException { - // TODO - optimize dstore implementation to do mass queries then wait for last result - - - List files = new ArrayList(); - for (int i = 0; i < remoteParents.length; i++) - { - IHostFile[] result = list(remoteParents[i], fileFilters[i], fileType, monitor); - for (int j = 0; j < result.length; j++) - { - files.add(result[j]); - } - } - - return (IHostFile[])files.toArray(new IHostFile[files.size()]); + String queryString = getQueryString(fileType); + + return fetchMulti(remoteParents, fileFilters, queryString, monitor); } @@ -1412,6 +1422,57 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer DataElement[] results = dsQueryCommand(deObj, args, queryType, monitor); return convertToHostFiles(results, fileFilter); } + + /** + * Fetch multiple results (for different parents an filters + * + * @param remoteParents the parents to query + * @param fileFilters the filters for each parent to query + * @param queryType the type of query (files, folders, both, etc) + * @param monitor the progress monitor + * @return the results + */ + protected IHostFile[] fetchMulti(String[] remoteParents, String[] fileFilters, String queryType, IProgressMonitor monitor) + { + DataStore ds = getDataStore(); + if (ds == null) + { + return new IHostFile[0]; + } + + ArrayList[] argses = new ArrayList[remoteParents.length]; + DataElement subjects[] = new DataElement[remoteParents.length]; + + for (int i = 0; i < remoteParents.length; i++) + { + // create filter descriptor + DataElement deObj = getElementFor(remoteParents[i]); + if (deObj == null) + { + DataElement universaltemp = getMinerElement(); + deObj = ds.createObject(universaltemp, IUniversalDataStoreConstants.UNIVERSAL_FILTER_DESCRIPTOR, remoteParents[i], remoteParents[i], "", false); //$NON-NLS-1$ + } + subjects[i] = deObj; + + DataElement attributes = getAttributes(fileFilters[i], true); + ArrayList args = new ArrayList(1); + args.add(attributes); + argses[i] = args; + } + + List consolidatedResults = dsQueryCommandMulti(subjects, argses, queryType, monitor); + List convertedResults = new ArrayList(); + for (int r = 0; r < consolidatedResults.size(); r++) + { + IHostFile[] results = convertToHostFiles((DataElement[])consolidatedResults.get(r), fileFilters[r]); + for (int c = 0; c < results.length; c++) + { + convertedResults.add(results[c]); + } + } + + return (IHostFile[])convertedResults.toArray(new IHostFile[consolidatedResults.size()]); + } public boolean isCaseSensitive() { diff --git a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/AbstractDStoreService.java b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/AbstractDStoreService.java index 7d212a78662..c39cab262c5 100644 --- a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/AbstractDStoreService.java +++ b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/AbstractDStoreService.java @@ -84,6 +84,19 @@ public abstract class AbstractDStoreService implements IDStoreService } } + protected DataElement[] dsQueryCommand(DataElement subject, String command, IProgressMonitor monitor) + { + return dsQueryCommand(subject, null, command, monitor); + } + + /** + * query + * @param subject + * @param args + * @param command + * @param monitor + * @return + */ protected DataElement[] dsQueryCommand(DataElement subject, ArrayList args, String command, IProgressMonitor monitor) { // query roots @@ -92,7 +105,16 @@ public abstract class AbstractDStoreService implements IDStoreService if (queryCmd != null && ds != null) { - DataElement status = ds.command(queryCmd, args, subject, true); + DataElement status = null; + + if (args != null) + { + status = ds.command(queryCmd, args, subject, true); + } + else + { + status = ds.command(queryCmd, subject, true); + } try { DStoreStatusMonitor smon = getStatusMonitor(getDataStore()); @@ -124,6 +146,88 @@ public abstract class AbstractDStoreService implements IDStoreService return new DataElement[0]; } + protected List dsQueryCommandMulti(DataElement[] subjects, String command, IProgressMonitor monitor) + { + return dsQueryCommandMulti(subjects, null, command, monitor); + } + + /** + * Query multiple subjects in one shot + * @param subjects the subjects to query + * @param command the query command + * @param args args for the command - may be null + * @param monitor the progress monitor + * @return a list of DataElement[]s containing the results of each query + */ + protected List dsQueryCommandMulti(DataElement[] subjects, ArrayList[] argses, String command, IProgressMonitor monitor) + { + List statuses = new ArrayList(); + DataStore ds = getDataStore(); + DStoreStatusMonitor smon = getStatusMonitor(ds); + + + for (int i = 0; i < subjects.length && !monitor.isCanceled(); i++) + { + DataElement subject = subjects[i]; + + DataElement queryCmd = getCommandDescriptor(subject, command); + if (queryCmd != null && ds != null) + { + DataElement status = null; + if (argses != null) + { + status = ds.command(queryCmd, argses[i], subject, true); + } + else + { + status = ds.command(queryCmd, subject, true); + } + statuses.add(status); + } + } + + List consolidatedResults = new ArrayList(); + + // wait for each command to complete + for (int i = 0; i < statuses.size() && !monitor.isCanceled(); i++) + { + DataElement status = (DataElement)statuses.get(i); + DataElement deObj = subjects[i]; + + try + { + smon.waitForUpdate(status, monitor); + + if (!monitor.isCanceled() && smon.determineStatusDone(status)) + { + List nested = deObj.getNestedData(); + if (nested != null) + { + consolidatedResults.add(nested.toArray(new DataElement[nested.size()])); + } + } + } + catch (InterruptedException e) + { + // cancel monitor if it's still not canceled + if (monitor != null && !monitor.isCanceled()) + { + monitor.setCanceled(true); + } + + //InterruptedException is used to report user cancellation, so no need to log + //This should be reviewed (use OperationCanceledException) with bug #190750 + } + } + + return consolidatedResults; + } + + + + + + protected DataElement dsStatusCommand(DataElement subject, ArrayList args, String command, IProgressMonitor monitor) { // query roots @@ -154,39 +258,6 @@ public abstract class AbstractDStoreService implements IDStoreService return null; } - protected DataElement[] dsQueryCommand(DataElement subject, String command, IProgressMonitor monitor) - { - // query roots - DataElement queryCmd = getCommandDescriptor(subject, command); - DataStore ds = getDataStore(); - if (queryCmd != null && ds != null) - { - DataElement status = ds.command(queryCmd, subject, true); - try - { - getStatusMonitor(ds).waitForUpdate(status, monitor); - checkHostJVM(); - // get results - List nested = subject.getNestedData(); - if (nested != null) - { - return (DataElement[])nested.toArray(new DataElement[subject.getNestedSize()]); - } - } - catch (InterruptedException e) - { - // cancel monitor if it's still not canceled - if (monitor != null && !monitor.isCanceled()) - { - monitor.setCanceled(true); - } - - //InterruptedException is used to report user cancellation, so no need to log - //This should be reviewed (use OperationCanceledException) with bug #190750 - } - } - return new DataElement[0]; - } protected DataElement dsStatusCommand(DataElement subject, String command, IProgressMonitor monitor) { diff --git a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/util/DStoreStatusMonitor.java b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/util/DStoreStatusMonitor.java index 8b9455fd442..3a7ea0d0ce1 100644 --- a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/util/DStoreStatusMonitor.java +++ b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/util/DStoreStatusMonitor.java @@ -290,13 +290,15 @@ public class DStoreStatusMonitor implements IDomainListener // Prevent infinite looping by introducing a threshold for wait - int WaitThreshold = 600; //default. sleep(100ms) for 600 times + int WaitThreshold = 200; //default. sleep(100ms) for 60 times if ( wait > 0 ) WaitThreshold = wait*10; // 1 second means 10 sleep(100ms) else if ( wait == -1 ) // force a diagnostic WaitThreshold = -1; - + int nudges = 0; // nudges used for waking up server with slow connections + // nudge up to 12 times before giving up + if (display != null) { // Current thread is UI thread @@ -370,8 +372,13 @@ public class DStoreStatusMonitor implements IDomainListener if (WaitThreshold == 0) { + wakeupServer(status); + // no diagnostic factory but there is a timeout - return status; // returning the undone status object + if (nudges >= 12) + return status; // returning the undone status object + + nudges++; } else if (_networkDown) { @@ -386,6 +393,19 @@ public class DStoreStatusMonitor implements IDomainListener return status; } + private void wakeupServer(DataElement status) + { + if (status != null) + { + // token command to wake up update handler + DataElement cmdDescriptor = _dataStore.findCommandDescriptor("C_REFRESH"); + DataElement subject = (DataElement)status.getParent().get(0); + if (cmdDescriptor != null) + { + _dataStore.command(cmdDescriptor, subject); + } + } + } /** * Causes the current thread to wait until this class request has been