Bug 532420 - Make Container Core Build indexing more efficient

- add new ICBuildConfiguration2 to keep API checks happy
- remove refreshScannerInfo method from ICBuildConfiguration and
  put it in ICBuildConfiguration2
- make CBuildConfiguration implement ICBuildConfiguration2
- update ContainerPropertyVolumesModel to use new Docker plug-ins
  using docker-client 8.9.2.
- fix MesonBuildConfiguration to use a job for each compile line being
  processed, then wait until all jobs are done before causing
  an reindex to occur (this will maximize parallelism when building
  in Containers)
- fix ContainerCommandLauncherFactory to save the project so we
  can exclude project directories when copying header files using
  the new Docker Tooling interfaces
- fix CoreBuildLaunchBarTracker to use ICBuildConfiguration2
  interface to make the call to refreshScannerInfo


Change-Id: I2138f5111614e7821e46c22731397a01035eac0a

build/org.eclipse.cdt.meson.core/src/org/eclipse/cdt/internal/meson/core/MesonBuildConfiguration.java

@@ -336,6 +336,7 @@ public class MesonBuildConfiguration extends CBuildConfiguration {
 		IProject project = getProject();
 		Path commandsFile = getBuildDirectory().resolve("compile_commands.json"); //$NON-NLS-1$
 		if (Files.exists(commandsFile)) {
+			List<Job> jobsList = new ArrayList<>();
 			monitor.setTaskName(Messages.MesonBuildConfiguration_ProcCompJson);
 			try (FileReader reader = new FileReader(commandsFile.toFile())) {
 				Gson gson = new Gson();
@@ -345,7 +346,14 @@ public class MesonBuildConfiguration extends CBuildConfiguration {
 					dedupedCmds.put(command.getFile(), command);
 				}
 				for (CompileCommand command : dedupedCmds.values()) {
-					processLine(command.getCommand());
+					processLine(command.getCommand(), jobsList);
+				}
+				for (Job j : jobsList) {
+					try {
+						j.join();
+					} catch (InterruptedException e) {
+						// ignore
+					}
 				}
 				shutdown();
 			} catch (IOException e) {

core/org.eclipse.cdt.core/.settings/.api_filters

@@ -1,12 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <component id="org.eclipse.cdt.core" version="2">
     <resource path="src/org/eclipse/cdt/core/build/ICBuildConfiguration.java" type="org.eclipse.cdt.core.build.ICBuildConfiguration">
-        <filter id="403767336">
-            <message_arguments>
-                <message_argument value="org.eclipse.cdt.core.build.ICBuildConfiguration"/>
-                <message_argument value="TOOLCHAIN_ID"/>
-            </message_arguments>
-        </filter>
         <filter id="403767336">
             <message_arguments>
                 <message_argument value="org.eclipse.cdt.core.build.ICBuildConfiguration"/>

core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/CBuildConfiguration.java

@@ -80,6 +80,7 @@ import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.PlatformObject;
 import org.eclipse.core.runtime.Status;
+import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.osgi.service.prefs.BackingStoreException;
 import org.osgi.service.prefs.Preferences;
@@ -100,7 +101,8 @@ import com.google.gson.JsonParseException;
  * @since 6.0
  */
 public abstract class CBuildConfiguration extends PlatformObject
-		implements ICBuildConfiguration, IMarkerGenerator, IConsoleParser, IElementChangedListener {
+		implements ICBuildConfiguration, ICBuildConfiguration2, IMarkerGenerator, 
+		IConsoleParser, IElementChangedListener {
 
 	private static final String LAUNCH_MODE = "cdt.launchMode"; //$NON-NLS-1$
 
@@ -110,6 +112,8 @@ public abstract class CBuildConfiguration extends PlatformObject
 	private final IBuildConfiguration config;
 	private final IToolChain toolChain;
 	private String launchMode;
+	
+	private Object scannerInfoLock = new Object();
 
 	private final Map<IResource, List<IScannerInfoChangeListener>> scannerInfoListeners = new HashMap<>();
 	private ScannerInfoCache scannerInfoCache;
@@ -613,31 +617,33 @@ public abstract class CBuildConfiguration extends PlatformObject
 	/**
 	 * @since 6.1
 	 */
-	protected synchronized void loadScannerInfoCache() {
-		if (scannerInfoCache == null) {
-			File cacheFile = getScannerInfoCacheFile();
-			if (cacheFile.exists()) {
-				try (FileReader reader = new FileReader(cacheFile)) {
-					GsonBuilder gsonBuilder = new GsonBuilder();
-					gsonBuilder.registerTypeAdapter(IExtendedScannerInfo.class,
-							new IExtendedScannerInfoCreator());
-					Gson gson = gsonBuilder.create();
-					scannerInfoCache = gson.fromJson(reader, ScannerInfoCache.class);
-				} catch (IOException e) {
-					CCorePlugin.log(e);
+	protected void loadScannerInfoCache() {
+		synchronized (scannerInfoLock) {
+			if (scannerInfoCache == null) {
+				File cacheFile = getScannerInfoCacheFile();
+				if (cacheFile.exists()) {
+					try (FileReader reader = new FileReader(cacheFile)) {
+						GsonBuilder gsonBuilder = new GsonBuilder();
+						gsonBuilder.registerTypeAdapter(IExtendedScannerInfo.class,
+								new IExtendedScannerInfoCreator());
+						Gson gson = gsonBuilder.create();
+						scannerInfoCache = gson.fromJson(reader, ScannerInfoCache.class);
+					} catch (IOException e) {
+						CCorePlugin.log(e);
+						scannerInfoCache = new ScannerInfoCache();
+					}
+				} else {
 					scannerInfoCache = new ScannerInfoCache();
 				}
-			} else {
-				scannerInfoCache = new ScannerInfoCache();
+				scannerInfoCache.initCache();
 			}
-			scannerInfoCache.initCache();
 		}
 	}
 
 	/**
 	 * @since 6.1
 	 */
-	protected void saveScannerInfoCache() {
+	protected synchronized void saveScannerInfoCache() {
 		File cacheFile = getScannerInfoCacheFile();
 		if (!cacheFile.getParentFile().exists()) {
 			try {
@@ -650,7 +656,9 @@ public abstract class CBuildConfiguration extends PlatformObject
 
 		try (FileWriter writer = new FileWriter(getScannerInfoCacheFile())) {
 			Gson gson = new Gson();
-			gson.toJson(scannerInfoCache, writer);
+			synchronized (scannerInfoLock) {
+				gson.toJson(scannerInfoCache, writer);
+			}
 		} catch (IOException e) {
 			CCorePlugin.log(e);
 		}
@@ -694,7 +702,10 @@ public abstract class CBuildConfiguration extends PlatformObject
 	@Override
 	public IScannerInfo getScannerInformation(IResource resource) {
 		loadScannerInfoCache();
-		IExtendedScannerInfo info = scannerInfoCache.getScannerInfo(resource);
+		IExtendedScannerInfo info = null;
+		synchronized (scannerInfoLock) {
+			info = scannerInfoCache.getScannerInfo(resource);
+		}
 		if (info == null) {
 			ICElement celement = CCorePlugin.getDefault().getCoreModel().create(resource);
 			if (celement instanceof ITranslationUnit) {
@@ -702,7 +713,9 @@ public abstract class CBuildConfiguration extends PlatformObject
 					ITranslationUnit tu = (ITranslationUnit) celement;
 					info = getToolChain().getDefaultScannerInfo(getBuildConfiguration(),
 							getBaseScannerInfo(resource), tu.getLanguage(), getBuildDirectoryURI());
-					scannerInfoCache.addScannerInfo(DEFAULT_COMMAND, info, resource);
+					synchronized (scannerInfoLock) {
+						scannerInfoCache.addScannerInfo(DEFAULT_COMMAND, info, resource);
+					}
 					saveScannerInfoCache();
 				} catch (CoreException e) {
 					CCorePlugin.log(e.getStatus());
@@ -731,12 +744,14 @@ public abstract class CBuildConfiguration extends PlatformObject
 				IResource resource = delta.getElement().getResource();
 				if (resource.getProject().equals(getProject())) {
 					loadScannerInfoCache();
-					if (scannerInfoCache.hasResource(DEFAULT_COMMAND, resource)) {
-						scannerInfoCache.removeResource(resource);
-					} else {
-						// Clear the whole command and exit the delta
-						scannerInfoCache.removeCommand(DEFAULT_COMMAND);
-						return;
+					synchronized (scannerInfoLock) {
+						if (scannerInfoCache.hasResource(DEFAULT_COMMAND, resource)) {
+							scannerInfoCache.removeResource(resource);
+						} else {
+							// Clear the whole command and exit the delta
+							scannerInfoCache.removeCommand(DEFAULT_COMMAND);
+							return;
+						}
 					}
 				}
 			}
@@ -852,19 +867,27 @@ public abstract class CBuildConfiguration extends PlatformObject
 
 				for (IResource resource : resources) {
 					loadScannerInfoCache();
-					if (scannerInfoCache.hasCommand(commandStrings)) {
-						if (!scannerInfoCache.hasResource(commandStrings, resource)) {
-							scannerInfoCache.addResource(commandStrings, resource);
-							infoChanged = true;
+					boolean hasCommand = true;
+					synchronized (scannerInfoLock) {
+						if (scannerInfoCache.hasCommand(commandStrings)) {
+							if (!scannerInfoCache.hasResource(commandStrings, resource)) {
+								scannerInfoCache.addResource(commandStrings, resource);
+								infoChanged = true;
+							}
+						} else {
+							hasCommand = false;
 						}
-					} else {
+					}
+					if (!hasCommand) {
 						Path commandPath = findCommand(command.get(0));
 						if (commandPath != null) {
 							command.set(0, commandPath.toString());
 							IExtendedScannerInfo info = getToolChain().getScannerInfo(getBuildConfiguration(),
 									command, null, resource, getBuildDirectoryURI());
-							scannerInfoCache.addScannerInfo(commandStrings, info, resource);
-							infoChanged = true;
+							synchronized (scannerInfoLock) {
+								scannerInfoCache.addScannerInfo(commandStrings, info, resource);
+								infoChanged = true;
+							}
 						}
 					}
 				}
@@ -877,7 +900,125 @@ public abstract class CBuildConfiguration extends PlatformObject
 			return false;
 		}
 	}
+
+	private class ScannerInfoJob extends Job {
+		private IToolChain toolchain;
+		private List<String> command;
+		private List<String> commandStrings;
+		private IResource resource;
+		private URI buildDirectoryURI;
+		
+		public ScannerInfoJob(String msg, IToolChain toolchain, List<String> command, IResource resource,
+				URI buildDirectoryURI, List<String> commandStrings) {
+			super(msg);
+			this.toolchain = toolchain;
+			this.command = command;
+			this.commandStrings = commandStrings;
+			this.resource = resource;
+			this.buildDirectoryURI = buildDirectoryURI;
+		}
+		
+		@Override
+		protected IStatus run(IProgressMonitor monitor) {
+			IExtendedScannerInfo info = toolchain.getScannerInfo(getBuildConfiguration(),
+					command, null, resource, buildDirectoryURI);
+			synchronized (scannerInfoLock) {
+				scannerInfoCache.addScannerInfo(commandStrings, info, resource);
+				infoChanged = true;
+			}
+			return Status.OK_STATUS;
+		}
+	}
 	
+	/**
+	 * Process a compile line for Scanner info in a separate job
+	 * 
+	 * @param line - line to process
+	 * @param jobsArray - array of Jobs to keep track of open scanner info jobs
+	 * @return - true if line processed, false otherwise
+	 * 
+	 * @since 6.5
+	 */
+	protected boolean processLine(String line, List<Job> jobsArray) {
+		// Split line into args, taking into account quotes
+		List<String> command = stripArgs(line);
+		
+		// Make sure it's a compile command
+		String[] compileCommands = toolChain.getCompileCommands();
+		boolean found = false;
+		loop:
+		for (String arg : command) {
+			// TODO we should really ask the toolchain, not all args start with '-'
+			if (arg.startsWith("-")) { //$NON-NLS-1$
+				// option found, missed our command
+				return false;
+			}
+
+			for (String cc : compileCommands) {
+				if (arg.endsWith(cc)
+						&& (arg.equals(cc) || arg.endsWith("/" + cc) || arg.endsWith("\\" + cc))) { //$NON-NLS-1$ //$NON-NLS-2$
+					found = true;
+					break loop;
+				}
+			}
+
+
+			if (Platform.getOS().equals(Platform.OS_WIN32) && !arg.endsWith(".exe")) { //$NON-NLS-1$
+				// Try with exe
+				arg = arg + ".exe"; //$NON-NLS-1$
+				for (String cc : compileCommands) {
+					if (arg.endsWith(cc)
+							&& (arg.equals(cc) || arg.endsWith("/" + cc) || arg.endsWith("\\" + cc))) { //$NON-NLS-1$ //$NON-NLS-2$
+						found = true;
+						break loop;
+					}
+				}
+			}
+		}
+
+		if (!found) {
+			return false;
+		}
+
+		try {
+			IResource[] resources = toolChain.getResourcesFromCommand(command, getBuildDirectoryURI());
+			if (resources != null && resources.length > 0) {
+				List<String> commandStrings = toolChain.stripCommand(command, resources);
+
+				for (IResource resource : resources) {
+					loadScannerInfoCache();
+					boolean hasCommand = true;
+					synchronized (scannerInfoLock) {
+						if (scannerInfoCache.hasCommand(commandStrings)) {
+							if (!scannerInfoCache.hasResource(commandStrings, resource)) {
+								scannerInfoCache.addResource(commandStrings, resource);
+								infoChanged = true;
+							}
+						} else {
+							hasCommand = false;
+						}
+					}
+					if (!hasCommand) {
+						Path commandPath = findCommand(command.get(0));
+						if (commandPath != null) {
+							command.set(0, commandPath.toString());
+							Job job = new ScannerInfoJob(String.format(Messages.CBuildConfiguration_RunningScannerInfo, resource), 
+									getToolChain(), command, resource, getBuildDirectoryURI(), commandStrings);
+							job.schedule();
+							jobsArray.add(job);
+						}
+					}
+				}
+				return true;
+			} else {
+				return false;
+			}
+		} catch (CoreException e) {
+			CCorePlugin.log(e);
+			return false;
+		}
+	}
+
 	/**
 	 * @since 6.5
 	 */

core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ICBuildConfiguration.java

@@ -125,13 +125,6 @@ public interface ICBuildConfiguration extends IAdaptable, IScannerInfoProvider {
 	 */
 	void clean(IConsole console, IProgressMonitor monitor) throws CoreException;
 	
-	/**
-	 * Refresh the Scanner info
-	 * 
-	 * @since 6.5
-	 */
-	void refreshScannerInfo() throws CoreException;
-
 	/**
 	 * The binaries produced by the build.
 	 * 

core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ICBuildConfiguration2.java

@@ -0,0 +1,25 @@
+/*******************************************************************************
+ * Copyright (c) 2018 Red Hat Inc. and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ * 
+ * Contributors:
+ * 		Red Hat Inc. - initial implementation
+ *******************************************************************************/
+package org.eclipse.cdt.core.build;
+
+import org.eclipse.core.runtime.CoreException;
+
+/**
+ * @since 6.5
+ */
+public interface ICBuildConfiguration2 {
+
+	/**
+	 * Refresh the Scanner info
+	 */
+	void refreshScannerInfo() throws CoreException;
+
+}

core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/core/build/Messages.java

@@ -14,6 +14,7 @@ public class Messages extends NLS {
 	public static String CBuildConfiguration_CreateJob;
 	public static String CBuildConfiguration_ToolchainMissing;
 	public static String CBuildConfiguration_Location;
+	public static String CBuildConfiguration_RunningScannerInfo;
 	public static String CBuilder_ExceptionWhileBuilding;
 	public static String CBuilder_ExceptionWhileBuilding2;
 	public static String CBuilder_NotConfiguredCorrectly;

core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/core/build/messages.properties

@@ -15,3 +15,4 @@ StandardBuildConfiguration_CommandNotFound=Error: build command '%s' not found
 CBuildConfiguration_CreateJob=Create Build Folder
 CBuildConfiguration_Location=line %d, external location: %s
 CBuildConfiguration_ToolchainMissing=Toolchain is missing for build configuration
+CBuildConfiguration_RunningScannerInfo=Calculating scanner info for %s

debug/org.eclipse.cdt.debug.core/src/org/eclipse/cdt/debug/core/launch/CoreBuildLaunchBarTracker.java

@@ -12,6 +12,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import org.eclipse.cdt.core.build.ICBuildConfiguration;
+import org.eclipse.cdt.core.build.ICBuildConfiguration2;
 import org.eclipse.cdt.core.build.ICBuildConfigurationManager;
 import org.eclipse.cdt.core.build.IToolChain;
 import org.eclipse.cdt.core.build.IToolChainManager;
@@ -131,7 +132,7 @@ public class CoreBuildLaunchBarTracker implements ILaunchBarListener {
 							desc.setActiveBuildConfig(buildConfig.getBuildConfiguration().getName());
 							finalProject.setDescription(desc, monitor);
 							// build config has changed so Scanner Info may change too which would affect indexing
-							buildConfig.refreshScannerInfo();
+							((ICBuildConfiguration2)buildConfig).refreshScannerInfo();
 						}
 					}
 

launch/org.eclipse.cdt.docker.launcher/src/org/eclipse/cdt/docker/launcher/ContainerCommandLauncherFactory.java

@@ -41,8 +41,11 @@ import org.eclipse.linuxtools.docker.ui.launch.ContainerLauncher;
 public class ContainerCommandLauncherFactory
 		implements ICommandLauncherFactory, ICommandLauncherFactory2 {
 
+	private IProject project;
+
 	@Override
 	public ICommandLauncher getCommandLauncher(IProject project) {
+		this.project = project;
 		// check if container build enablement has been checked
 		ICConfigurationDescription cfgd = CoreModel.getDefault()
 				.getProjectDescription(project)
@@ -80,6 +83,7 @@ public class ContainerCommandLauncherFactory
 		if (cfg == null) {
 			return null;
 		}
+		this.project = (IProject) cfg.getManagedProject().getOwner();
 		IOptionalBuildProperties props = cfg.getOptionalBuildProperties();
 		if (props != null) {
 			String enablementProperty = props.getProperty(
@@ -99,6 +103,11 @@ public class ContainerCommandLauncherFactory
 
 	@Override
 	public ICommandLauncher getCommandLauncher(ICBuildConfiguration cfgd) {
+		try {
+			this.project = cfgd.getBuildConfiguration().getProject();
+		} catch (CoreException e1) {
+			return null;
+		}
 		// check if container linux os is set
 		IToolChain toolchain;
 		try {
@@ -201,10 +210,12 @@ public class ContainerCommandLauncherFactory
 							return;
 						}
 						IPath hostDir = pluginPath;
+						List<String> excludeList = new ArrayList<>();
+						excludeList.add(project.getLocation().toString());
 						@SuppressWarnings("unused")
 						int status = launcher.fetchContainerDirs(connectionName,
 								imageName,
-								paths, hostDir);
+								paths, excludeList, hostDir);
 					}
 				}
 			}
@@ -294,8 +305,11 @@ public class ContainerCommandLauncherFactory
 					return includePaths;
 				}
 				IPath hostDir = pluginPath;
+				// exclude project directories from any copying operation
+				List<String> excludeList = new ArrayList<>();
+				excludeList.add(project.getLocation().toString());
 				int status = launcher.fetchContainerDirsSync(connectionName,
-						imageName, includePaths, hostDir);
+						imageName, includePaths, excludeList, hostDir);
 				if (status == 0) {
 					Set<String> copiedVolumes = launcher
 							.getCopiedVolumes(connectionName, imageName);

launch/org.eclipse.cdt.docker.launcher/src/org/eclipse/cdt/internal/docker/launcher/ui/launchbar/ContainerGCCToolChain.java

@@ -383,8 +383,6 @@ public class ContainerGCCToolChain extends PlatformObject implements IToolChain
 				return null;
 			}
 
-			// process.waitFor();
-
 			// Scan for the scanner info
 			Map<String, String> symbols = new HashMap<>();
 			List<String> includePath = new ArrayList<>();