mirrors/cdt
1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-04-21 21:52:10 +02:00
Code Issues Wiki Activity

Bug 565553 - Improve performance of build command parsers with large number of files

Browse source 
Cache results of various path resolution algorithms.

Resolving paths is particularly slow while creating entries, see
AbstractLanguageSettingsOutputScanner.createResolvedPathEntry.

There are three main callees within that method that this patch addresses with
a caching approach:

* findContainerForLocationURI: First, this finds containers for a given URI in
the workspace using Eclipse resources API. Then a single container is
selected based on a preferred project. This can done repeatedly for include
paths, which are often similar for source files in a given project or source
folder. This first step is the expensive one and it only depends on one
argument (the URI) and a simple IResource[] return type, so the cache here is
done for this operation. Then the post-filtering is kept as is.

* findFileForLocationURI: Similar to the container case but for files. A
typical projet has much less file paths than folder paths in its options. One
more common option using file paths is -include. The same approach is applied
here as the previous point because there are performance gains but they are
smaller if you consider typical projet setup.

* findBestFitInWorkspace: When a path cannot be found, this makes an attempt to
find the parsed path relative to every folder of the workspace, by starting
first with the preferred project, then its referenced projects and then the
rest. Caching the result of findBestFitInWorkspace itself is too cumbersome
because the result depends on 3 variables (currentProject,
currentCfgDescription and parsedName) which would make a complex cache key.
Instead, caching the result of findPathInFolder at the project level is
sufficient, with little to no performance difference.

In all three cases, the class LRUCache is used in order to limit memory
consumption of the cache. A limit of 100 elements for each cache was chosen
based on experimentation with a few projects like LLVM and projets several
times bigger. A limit higher than necessary for small projects does not incur a
noticeable overhead for small projects and a limit too small for very large
projects merely diminishes the performance gains.

Using LLVM code base as a test, the time to parse options for all files:
Before: 68395ms, after: 5599ms

Change-Id: Ib997e9373087950f9ae6d93bbb1a5f265431c6bc
Signed-off-by: Marc-Andre Laperle <malaperle@gmail.com>
This commit is contained in:
Marc-Andre Laperle 2020-07-31 16:21:52 -04:00 • committed by Marc-André Laperle
parent 1cb1233c33
commit 9e7b5beaa9
1 changed files with 37 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/language/settings/providers/AbstractLanguageSettingsOutputScanner.java
View file

@ -19,6 +19,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
@ -40,6 +41,7 @@ import org.eclipse.cdt.core.settings.model.ICConfigurationDescription;
import org.eclipse.cdt.core.settings.model.ICLanguageSettingEntry;
import org.eclipse.cdt.core.settings.model.ICSettingEntry;
import org.eclipse.cdt.core.settings.model.util.CDataUtil;
import org.eclipse.cdt.internal.core.LRUCache;
import org.eclipse.cdt.internal.core.XmlUtil;
import org.eclipse.cdt.managedbuilder.core.ManagedBuilderCorePlugin;
import org.eclipse.cdt.utils.EFSExtensionManager;
@ -89,6 +91,14 @@ public abstract class AbstractLanguageSettingsOutputScanner extends LanguageSett
protected String parsedResourceName = null;
protected boolean isResolvingPaths = true;
private static final int FIND_RESOURCES_CACHE_SIZE = 100;
private LRUCache<URI, IResource[]> workspaceRootFindContainersForLocationURICache = new LRUCache<>(
FIND_RESOURCES_CACHE_SIZE);
private LRUCache<URI, IResource[]> workspaceRootFindFilesForLocationURICache = new LRUCache<>(
FIND_RESOURCES_CACHE_SIZE);
private HashMap<IProject, LRUCache<IPath, List<IResource>>> findPathInProjectCache = new HashMap<>();
/** @since 8.2 */
protected EFSExtensionProvider efsProvider = null;
@ -438,6 +448,13 @@ public abstract class AbstractLanguageSettingsOutputScanner extends LanguageSett
currentLanguageId = null;
currentResource = null;
cwdTracker = null;
clearCaches();
}
private void clearCaches() {
workspaceRootFindContainersForLocationURICache.clear();
workspaceRootFindFilesForLocationURICache.clear();
findPathInProjectCache.clear();
}
@Override
@ -671,15 +688,16 @@ public abstract class AbstractLanguageSettingsOutputScanner extends LanguageSett
* Find file resource in the workspace for a given URI with a preference for the resource
* to reside in the given project.
*/
private static IResource findFileForLocationURI(URI uri, IProject preferredProject, boolean checkExistence) {
private IResource findFileForLocationURI(URI uri, IProject preferredProject, boolean checkExistence) {
if (!uri.isAbsolute()) {
// IWorkspaceRoot.findFilesForLocationURI(URI) below requires an absolute URI
// therefore we haven't/aren't going to find the file based on this URI.
return null;
}
IResource sourceFile = null;
IWorkspaceRoot root = ResourcesPlugin.getWorkspace().getRoot();
IResource[] resources = root.findFilesForLocationURI(uri);
IResource[] resources = workspaceRootFindFilesForLocationURICache.computeIfAbsent(uri,
key -> ResourcesPlugin.getWorkspace().getRoot().findFilesForLocationURI(key));
for (IResource rc : resources) {
if (!checkExistence || rc.isAccessible()) {
if (rc.getProject().equals(preferredProject)) {
@ -698,9 +716,11 @@ public abstract class AbstractLanguageSettingsOutputScanner extends LanguageSett
* Return a resource in workspace corresponding the given folder {@link URI} preferable residing in
* the provided project.
*/
private static IResource findContainerForLocationURI(URI uri, IProject preferredProject, boolean checkExistence) {
private IResource findContainerForLocationURI(URI uri, IProject preferredProject, boolean checkExistence) {
IResource resource = null;
IResource[] resources = ResourcesPlugin.getWorkspace().getRoot().findContainersForLocationURI(uri);
IResource[] resources = workspaceRootFindContainersForLocationURICache.computeIfAbsent(uri,
key -> ResourcesPlugin.getWorkspace().getRoot().findContainersForLocationURI(key));
for (IResource rc : resources) {
if ((rc instanceof IProject || rc instanceof IFolder) && (!checkExistence || rc.isAccessible())) { // treat IWorkspaceRoot as non-workspace path
if (rc.getProject().equals(preferredProject)) {
@ -912,6 +932,15 @@ public abstract class AbstractLanguageSettingsOutputScanner extends LanguageSett
return uri;
}
/**
* Find all resources in the project which might be represented by relative path passed.
*/
private List<IResource> findPathInProject(IPath path, IProject project) {
LRUCache<IPath, List<IResource>> cache = findPathInProjectCache.computeIfAbsent(project,
key -> new LRUCache<>(FIND_RESOURCES_CACHE_SIZE));
return cache.computeIfAbsent(path, key -> findPathInFolder(path, project));
}
/**
* Find all resources in the folder which might be represented by relative path passed.
*/
@ -952,7 +981,7 @@ public abstract class AbstractLanguageSettingsOutputScanner extends LanguageSett
// prefer current project
if (currentProject != null) {
List<IResource> result = findPathInFolder(path, currentProject);
List<IResource> result = findPathInProject(path, currentProject);
int size = result.size();
if (size == 1) { // found the one
return result.get(0);
@ -969,7 +998,7 @@ public abstract class AbstractLanguageSettingsOutputScanner extends LanguageSett
for (String prjName : referencedProjectsNames) {
IProject prj = root.getProject(prjName);
if (prj.isOpen()) {
List<IResource> result = findPathInFolder(path, prj);
List<IResource> result = findPathInProject(path, prj);
int size = result.size();
if (size == 1 && rc == null) {
rc = result.get(0);
@ -991,7 +1020,7 @@ public abstract class AbstractLanguageSettingsOutputScanner extends LanguageSett
IResource rc = null;
for (IProject prj : projects) {
if (!prj.equals(currentProject) && !referencedProjectsNames.contains(prj.getName()) && prj.isOpen()) {
List<IResource> result = findPathInFolder(path, prj);
List<IResource> result = findPathInProject(path, prj);
int size = result.size();
if (size == 1 && rc == null) {
rc = result.get(0);