[207178] optimizations for listMulti and getFileMulti for DstoreFlieService

rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java

@@ -784,7 +784,7 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 		return true;
 	}
 
-	public IHostFile getFile(String remoteParent, String name, IProgressMonitor monitor)
+	private DataElement getSubjectFor(String remoteParent, String name)
 	{
 		DataElement de = null;
 		if (name.equals(".") && name.equals(remoteParent)) //$NON-NLS-1$
@@ -801,10 +801,27 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 			buf.append(name);
 			de = getElementFor(buf.toString());
 		}
+		return de;
+	}
+	
+	private DataElement[] getSubjectsFor(String[] remoteParents, String[] names)
+	{
+		List subjects = new ArrayList();
+		for (int i = 0; i < remoteParents.length; i++)
+		{
+			DataElement de = getSubjectFor(remoteParents[i], names[i]);
+			subjects.add(de);
+		}
+		return (DataElement[])subjects.toArray(new DataElement[subjects.size()]);
+	}
+	
+	public IHostFile getFile(String remoteParent, String name, IProgressMonitor monitor)
+	{
+		DataElement de = getSubjectFor(remoteParent, name);
 		
 		// with 207095, it's possible to get here unconnected such that there is no element	
 		if (de != null) {
-			dsQueryCommand(de, IUniversalDataStoreConstants.C_QUERY_GET_REMOTE_OBJECT, monitor);
+			dsQueryCommand(de, null,  IUniversalDataStoreConstants.C_QUERY_GET_REMOTE_OBJECT, monitor);
 			//getFile call should also need to convert this DataElement into a HostFile using
 			//convertToHostFile() call.  This way, this DataElement will be put into _fileMap.
 			return convertToHostFile(de);
@@ -814,16 +831,17 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 		}
 	}
 	
+	/**
+	 * Mass query of individual files
+	 */
 	public IHostFile[] getFileMulti(String remoteParents[], String names[], IProgressMonitor monitor) 
 		throws SystemMessageException
 	{
-		// TODO optimize dstore version of this to do mass queries then wait for last status
+		DataElement[] subjects = getSubjectsFor(remoteParents, names);
-		List results = new ArrayList();
+		
-		for (int i = 0; i < remoteParents.length; i++)
+		dsQueryCommandMulti(subjects, null, IUniversalDataStoreConstants.C_QUERY_GET_REMOTE_OBJECT, monitor);
-		{
+
-			results.add(getFile(remoteParents[i], names[i], monitor));
+		return convertToHostFiles(subjects, "*");		 //$NON-NLS-1$
-		}
-		return (IHostFile[])results.toArray(new IHostFile[results.size()]);
 	}
 
 	/**
@@ -1303,9 +1321,7 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 		return convertToHostFiles(results, "*"); //$NON-NLS-1$
 	}
 	
-	
+	private String getQueryString(int fileType)
-
-	public IHostFile[] list(String remoteParent, String fileFilter, int fileType, IProgressMonitor monitor)
 	{
 		String queryString = null;
 		switch (fileType)
@@ -1323,7 +1339,12 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 			queryString = IUniversalDataStoreConstants.C_QUERY_VIEW_ALL;
 			break;
 		}
+		return queryString;
+	}
 
+	public IHostFile[] list(String remoteParent, String fileFilter, int fileType, IProgressMonitor monitor)
+	{
+		String queryString = getQueryString(fileType);		
 		return fetch(remoteParent, fileFilter, queryString, monitor);
 	}
 
@@ -1332,20 +1353,9 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 			String[] fileFilters, int fileType, IProgressMonitor monitor)
 			throws SystemMessageException 
 	{
-		// TODO - optimize dstore implementation to do mass queries then wait for last result
+		String queryString = getQueryString(fileType);
 
-		
+		return fetchMulti(remoteParents, fileFilters, queryString, monitor);
-		List files = new ArrayList();
-		for (int i = 0; i < remoteParents.length; i++)
-		{
-			IHostFile[] result = list(remoteParents[i], fileFilters[i], fileType, monitor);
-			for (int j = 0; j < result.length; j++)
-			{
-				files.add(result[j]);
-			}
-		}
-		
-		return (IHostFile[])files.toArray(new IHostFile[files.size()]);
 	}
 	
 	
@@ -1413,6 +1423,57 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 		return convertToHostFiles(results, fileFilter);
 	}
 	
+	/**
+	 * Fetch multiple results (for different parents an filters
+	 * 
+	 * @param remoteParents the parents to query
+	 * @param fileFilters the filters for each parent to query
+	 * @param queryType the type of query (files, folders, both, etc)
+	 * @param monitor the progress monitor
+	 * @return the results
+	 */
+	protected IHostFile[] fetchMulti(String[] remoteParents, String[] fileFilters, String queryType, IProgressMonitor monitor)
+	{
+		DataStore ds = getDataStore();
+		if (ds == null)
+		{
+			return new IHostFile[0];
+		}
+		
+		ArrayList[] argses = new ArrayList[remoteParents.length];
+		DataElement subjects[] = new DataElement[remoteParents.length];
+		
+		for (int i = 0; i < remoteParents.length; i++)
+		{
+			// create filter descriptor
+			DataElement deObj = getElementFor(remoteParents[i]);
+			if (deObj == null)
+			{
+				DataElement universaltemp = getMinerElement();
+				deObj = ds.createObject(universaltemp, IUniversalDataStoreConstants.UNIVERSAL_FILTER_DESCRIPTOR, remoteParents[i], remoteParents[i], "", false); //$NON-NLS-1$
+			}
+			subjects[i] = deObj;
+			
+			DataElement attributes = getAttributes(fileFilters[i], true);
+			ArrayList args = new ArrayList(1);
+			args.add(attributes);
+			argses[i] = args;
+		}
+			
+		List consolidatedResults = dsQueryCommandMulti(subjects, argses, queryType, monitor);		
+		List convertedResults = new ArrayList();
+		for (int r = 0; r < consolidatedResults.size(); r++)
+		{
+			IHostFile[] results = convertToHostFiles((DataElement[])consolidatedResults.get(r), fileFilters[r]);
+			for (int c = 0; c < results.length; c++)
+			{
+				convertedResults.add(results[c]);
+			}
+		}
+		
+		return (IHostFile[])convertedResults.toArray(new IHostFile[consolidatedResults.size()]);
+	}
+
 	public boolean isCaseSensitive()
 	{
 		return true;

rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/AbstractDStoreService.java

@@ -84,6 +84,19 @@ public abstract class AbstractDStoreService implements IDStoreService
 		}
 	}
 	 
+	protected DataElement[] dsQueryCommand(DataElement subject, String command, IProgressMonitor monitor)
+	{
+		return dsQueryCommand(subject, null, command, monitor);
+	}
+
+	/**
+	 * query 
+	 * @param subject
+	 * @param args
+	 * @param command
+	 * @param monitor
+	 * @return
+	 */
 	protected DataElement[] dsQueryCommand(DataElement subject, ArrayList args, String command, IProgressMonitor monitor)
 	{
 		// query roots
@@ -92,7 +105,16 @@ public abstract class AbstractDStoreService implements IDStoreService
 	
 		if (queryCmd != null && ds != null)
 		{
-			DataElement status = ds.command(queryCmd, args, subject, true);
+			DataElement status = null;
+			
+			if (args != null)
+			{
+				status = ds.command(queryCmd, args, subject, true);
+			}
+			else
+			{
+				status = ds.command(queryCmd, subject, true);
+			}
 			try
 			{
 				DStoreStatusMonitor smon = getStatusMonitor(getDataStore());
@@ -124,6 +146,88 @@ public abstract class AbstractDStoreService implements IDStoreService
 		return new DataElement[0];
 	}
 	
+	protected List dsQueryCommandMulti(DataElement[] subjects, String command, IProgressMonitor monitor)
+	{
+		return dsQueryCommandMulti(subjects, null, command, monitor);
+	}
+	
+	/**
+	 * Query multiple subjects in one shot
+	 * @param subjects the subjects to query
+	 * @param command the query command
+	 * @param args args for the command - may be null
+	 * @param monitor the progress monitor
+	 * @return a list of DataElement[]s containing the results of each query
+	 */
+	protected List dsQueryCommandMulti(DataElement[] subjects, ArrayList[] argses, String command, IProgressMonitor monitor)
+	{
+		List statuses = new ArrayList();
+		DataStore ds = getDataStore();	
+		DStoreStatusMonitor smon = getStatusMonitor(ds);
+		
+	
+		for (int i = 0; i < subjects.length && !monitor.isCanceled(); i++)
+		{
+			DataElement subject = subjects[i];
+			
+			DataElement queryCmd = getCommandDescriptor(subject, command);
+			if (queryCmd != null && ds != null)
+			{
+				DataElement status = null;
+				if (argses != null)
+				{
+					status = ds.command(queryCmd, argses[i], subject, true);
+				}
+				else
+				{
+					status = ds.command(queryCmd, subject, true);
+				}
+				statuses.add(status);				
+			}
+		}
+
+		List consolidatedResults = new ArrayList();
+		
+		// wait for each command to complete
+		for (int i = 0; i < statuses.size() && !monitor.isCanceled(); i++)
+		{
+		    DataElement status = (DataElement)statuses.get(i);
+		    DataElement deObj = subjects[i];
+		    
+		    try
+		    {
+		    	smon.waitForUpdate(status, monitor);		 		    
+		    	
+			    if (!monitor.isCanceled() && smon.determineStatusDone(status))
+			    {
+			    	List nested = deObj.getNestedData();
+			    	if (nested != null)
+			    	{
+			    		consolidatedResults.add(nested.toArray(new DataElement[nested.size()]));
+			    	}			    	
+			    }
+			}
+		    catch (InterruptedException e)
+		    {				
+		    	// cancel monitor if it's still not canceled
+		    	if (monitor != null && !monitor.isCanceled())
+		    	{
+		    		monitor.setCanceled(true);
+		    	}
+			
+			//InterruptedException is used to report user cancellation, so no need to log
+			//This should be reviewed (use OperationCanceledException) with bug #190750
+		    }	
+		}
+		
+		return consolidatedResults;
+	}
+	
+	
+	
+
+	
+	
 	protected DataElement dsStatusCommand(DataElement subject, ArrayList args, String command, IProgressMonitor monitor)
 	{
 		// query roots
@@ -154,39 +258,6 @@ public abstract class AbstractDStoreService implements IDStoreService
 		return null;
 	}
 	
-	protected DataElement[] dsQueryCommand(DataElement subject, String command, IProgressMonitor monitor)
-	{
-		// query roots
-		DataElement queryCmd = getCommandDescriptor(subject, command);
-		DataStore ds = getDataStore();
-		if (queryCmd != null && ds != null)
-		{
-			DataElement status = ds.command(queryCmd, subject, true);
-			try
-			{
-				getStatusMonitor(ds).waitForUpdate(status, monitor);
-				checkHostJVM();
-				// get results
-				List nested = subject.getNestedData();
-				if (nested != null)
-				{
-					return (DataElement[])nested.toArray(new DataElement[subject.getNestedSize()]);
-				}
-			}
-			catch (InterruptedException e)
-			{				
-				// cancel monitor if it's still not canceled
-				if (monitor != null && !monitor.isCanceled())
-				{
-					monitor.setCanceled(true);
-				}
-				
-				//InterruptedException is used to report user cancellation, so no need to log
-				//This should be reviewed (use OperationCanceledException) with bug #190750
-			}			
-		}
-		return new DataElement[0];
-	}
 	
 	protected DataElement dsStatusCommand(DataElement subject, String command, IProgressMonitor monitor)
 	{

rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/services/dstore/util/DStoreStatusMonitor.java

@@ -290,12 +290,14 @@ public class DStoreStatusMonitor implements IDomainListener
 
 				
 	  // Prevent infinite looping by introducing a threshold for wait 
-      int WaitThreshold = 600; //default. sleep(100ms) for 600 times  		
+      int WaitThreshold = 200; //default. sleep(100ms) for 60 times  		
       if ( wait > 0 )
         WaitThreshold = wait*10; // 1 second means 10 sleep(100ms)
       else if ( wait == -1 ) // force a diagnostic
 	  		 WaitThreshold = -1;
          
+	  int nudges = 0; // nudges used for waking up server with slow connections
+	      // nudge up to 12 times before giving up
 	  
 		if (display != null) 
 		{
@@ -370,8 +372,13 @@ public class DStoreStatusMonitor implements IDomainListener
 
                    if (WaitThreshold == 0)
 				    {
+                     	wakeupServer(status);
+                        
 				        // no diagnostic factory but there is a timeout
+                    	if (nudges >= 12)
                     		return status;  // returning the undone status object
+                    	
+                    	nudges++;
 				    }
                     else if (_networkDown)
                     {
@@ -386,6 +393,19 @@ public class DStoreStatusMonitor implements IDomainListener
 		return status;
 	}
 
+	private void wakeupServer(DataElement status)
+	{
+		if (status != null)
+		{
+			// token command to wake up update handler
+			DataElement cmdDescriptor = _dataStore.findCommandDescriptor("C_REFRESH");
+			DataElement subject = (DataElement)status.getParent().get(0);
+			if (cmdDescriptor != null)
+			{
+				_dataStore.command(cmdDescriptor, subject);
+			}
+		}
+	}
 
 	/**
 	 * Causes the current thread to wait until this class request has been