From 6c6901547e50469c64bbae358d686e5f7c94a628 Mon Sep 17 00:00:00 2001 From: Doug Schaefer Date: Thu, 18 Aug 2016 17:02:57 -0400 Subject: [PATCH] Scanner discovery for CMake projects. Reads the compile_commands.json file and feeds the commands into the processLine method of the build config which creates the scanner info. The scanner info is cached in memory and stored in the metadata directory. Change-Id: I8b04e661dfe767904d1c10119c07167fee8cd7e4 --- .../cdt/build/gcc/core/GCCToolChain.java | 211 ++++++++++++----- .../META-INF/MANIFEST.MF | 3 +- .../cdt/cmake/core/CMakeProjectGenerator.java | 17 +- .../internal/CMakeBuildConfiguration.java | 41 +++- .../cmake/core/internal/CompileCommand.java | 28 +++ .../templates/simple/CMakeLists.txt | 2 +- .../templates/simple/manifest.xml | 3 +- .../org.eclipse.cdt.core/META-INF/MANIFEST.MF | 3 +- .../cdt/core/build/CBuildConfiguration.java | 217 +++++++++++++++--- .../eclipse/cdt/core/build/IToolChain.java | 199 +++++++++++++++- .../cdt/core/build/ScannerInfoCache.java | 126 ++++++++++ .../debug.product | 1 + 12 files changed, 728 insertions(+), 123 deletions(-) create mode 100644 build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/internal/CompileCommand.java create mode 100644 core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ScannerInfoCache.java diff --git a/build/org.eclipse.cdt.build.gcc.core/src/org/eclipse/cdt/build/gcc/core/GCCToolChain.java b/build/org.eclipse.cdt.build.gcc.core/src/org/eclipse/cdt/build/gcc/core/GCCToolChain.java index 9ffdc420673..8354d8651bb 100644 --- a/build/org.eclipse.cdt.build.gcc.core/src/org/eclipse/cdt/build/gcc/core/GCCToolChain.java +++ b/build/org.eclipse.cdt.build.gcc.core/src/org/eclipse/cdt/build/gcc/core/GCCToolChain.java @@ -16,7 +16,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,8 +26,11 @@ import org.eclipse.cdt.build.gcc.core.internal.Activator; import org.eclipse.cdt.core.CCorePlugin; import org.eclipse.cdt.core.build.IToolChain; import org.eclipse.cdt.core.build.IToolChainProvider; +import org.eclipse.cdt.core.dom.ast.gnu.c.GCCLanguage; +import org.eclipse.cdt.core.dom.ast.gnu.cpp.GPPLanguage; import org.eclipse.cdt.core.envvar.EnvironmentVariable; import org.eclipse.cdt.core.envvar.IEnvironmentVariable; +import org.eclipse.cdt.core.model.ILanguage; import org.eclipse.cdt.core.parser.ExtendedScannerInfo; import org.eclipse.cdt.core.parser.IExtendedScannerInfo; import org.eclipse.core.resources.IBuildConfiguration; @@ -57,8 +59,6 @@ public class GCCToolChain extends PlatformObject implements IToolChain { private final IEnvironmentVariable[] envVars; private final Map properties = new HashMap<>(); - protected String[] compileCommands; - public GCCToolChain(IToolChainProvider provider, String id, String version) { this(provider, id, version, null, null); } @@ -157,11 +157,12 @@ public class GCCToolChain extends PlatformObject implements IToolChain { } @Override - public IExtendedScannerInfo getScannerInfo(IBuildConfiguration buildConfig, Path command, String[] args, + public IExtendedScannerInfo getScannerInfo(IBuildConfiguration buildConfig, List commandStrings, IExtendedScannerInfo baseScannerInfo, IResource resource, URI buildDirectoryURI) { try { Path buildDirectory = Paths.get(buildDirectoryURI); + Path command = Paths.get(commandStrings.get(0)); List commandLine = new ArrayList<>(); if (command.isAbsolute()) { commandLine.add(command.toString()); @@ -176,7 +177,7 @@ public class GCCToolChain extends PlatformObject implements IToolChain { } addDiscoveryOptions(commandLine); - commandLine.addAll(Arrays.asList(args)); + commandLine.addAll(commandStrings.subList(1, commandStrings.size())); // Change output to stdout boolean haveOut = false; @@ -225,53 +226,113 @@ public class GCCToolChain extends PlatformObject implements IToolChain { commandLine.add(tmpFile.toString()); } - Files.createDirectories(buildDirectory); - - // Startup the command - ProcessBuilder processBuilder = new ProcessBuilder(commandLine).directory(buildDirectory.toFile()) - .redirectErrorStream(true); - CCorePlugin.getDefault().getBuildEnvironmentManager().setEnvironment(processBuilder.environment(), - buildConfig, true); - Process process = processBuilder.start(); - - // Scan for the scanner info - Map symbols = new HashMap<>(); - List includePath = new ArrayList<>(); - Pattern definePattern = Pattern.compile("#define (.*)\\s(.*)"); //$NON-NLS-1$ - boolean inIncludePaths = false; - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { - for (String line = reader.readLine(); line != null; line = reader.readLine()) { - if (inIncludePaths) { - if (line.equals("End of search list.")) { //$NON-NLS-1$ - inIncludePaths = false; - } else { - includePath.add(line.trim()); - } - } else if (line.startsWith("#define ")) { //$NON-NLS-1$ - Matcher matcher = definePattern.matcher(line); - if (matcher.matches()) { - symbols.put(matcher.group(1), matcher.group(2)); - } - } else if (line.equals("#include <...> search starts here:")) { //$NON-NLS-1$ - inIncludePaths = true; - } - } - } - - try { - process.waitFor(); - } catch (InterruptedException e) { - Activator.log(e); - } - Files.delete(tmpFile); - - return new ExtendedScannerInfo(symbols, includePath.toArray(new String[includePath.size()])); + return getScannerInfo(buildConfig, commandLine, buildDirectory, tmpFile); } catch (IOException e) { Activator.log(e); return null; } } + @Override + public IExtendedScannerInfo getDefaultScannerInfo(IBuildConfiguration buildConfig, + IExtendedScannerInfo baseScannerInfo, ILanguage language, URI buildDirectoryURI) { + try { + String[] commands = getCompileCommands(language); + if (commands == null || commands.length == 0) { + // no default commands + return null; + } + + Path buildDirectory = Paths.get(buildDirectoryURI); + + // Pick the first one + Path command = Paths.get(commands[0]); + List commandLine = new ArrayList<>(); + if (command.isAbsolute()) { + commandLine.add(command.toString()); + } else { + commandLine.add(getCommandPath(command).toString()); + } + + if (baseScannerInfo != null && baseScannerInfo.getIncludePaths() != null) { + for (String includePath : baseScannerInfo.getIncludePaths()) { + commandLine.add("-I" + includePath); //$NON-NLS-1$ + } + } + + addDiscoveryOptions(commandLine); + + // output to stdout + commandLine.add("-o"); //$NON-NLS-1$ + commandLine.add("-"); //$NON-NLS-1$ + + // Source is an empty tmp file + String extension; + if (GPPLanguage.ID.equals(language.getId())) { + extension = ".cpp"; + } else if (GCCLanguage.ID.equals(language.getId())) { + extension = ".c"; + } else { + // In theory we shouldn't get here + return null; + } + + Path tmpFile = Files.createTempFile(buildDirectory, ".sc", extension); //$NON-NLS-1$ + commandLine.add(tmpFile.toString()); + + return getScannerInfo(buildConfig, commandLine, buildDirectory, tmpFile); + } catch (IOException e) { + Activator.log(e); + return null; + } + } + + private IExtendedScannerInfo getScannerInfo(IBuildConfiguration buildConfig, List commandLine, + Path buildDirectory, Path tmpFile) throws IOException { + Files.createDirectories(buildDirectory); + + // Startup the command + ProcessBuilder processBuilder = new ProcessBuilder(commandLine).directory(buildDirectory.toFile()) + .redirectErrorStream(true); + CCorePlugin.getDefault().getBuildEnvironmentManager().setEnvironment(processBuilder.environment(), + buildConfig, true); + Process process = processBuilder.start(); + + // Scan for the scanner info + Map symbols = new HashMap<>(); + List includePath = new ArrayList<>(); + Pattern definePattern = Pattern.compile("#define (.*)\\s(.*)"); //$NON-NLS-1$ + boolean inIncludePaths = false; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + for (String line = reader.readLine(); line != null; line = reader.readLine()) { + if (inIncludePaths) { + if (line.equals("End of search list.")) { //$NON-NLS-1$ + inIncludePaths = false; + } else { + includePath.add(line.trim()); + } + } else if (line.startsWith("#define ")) { //$NON-NLS-1$ + Matcher matcher = definePattern.matcher(line); + if (matcher.matches()) { + symbols.put(matcher.group(1), matcher.group(2)); + } + } else if (line.equals("#include <...> search starts here:")) { //$NON-NLS-1$ + inIncludePaths = true; + } + } + } + + try { + process.waitFor(); + } catch (InterruptedException e) { + Activator.log(e); + } + Files.delete(tmpFile); + + return new ExtendedScannerInfo(symbols, includePath.toArray(new String[includePath.size()])); + + } + @Override public String[] getErrorParserIds() { return new String[] { "org.eclipse.cdt.core.GCCErrorParser", //$NON-NLS-1$ @@ -331,32 +392,40 @@ public class GCCToolChain extends PlatformObject implements IToolChain { @Override public String[] getCompileCommands() { - if (compileCommands == null) { - List cmds = new ArrayList<>(); - for (String cmd : new String[] { "gcc", "g++", "clang", "clang++" }) { //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ - cmd = prefix != null ? cmd : prefix + cmd; - Path cmdPath = getCommandPath(Paths.get(cmd)); - if (cmdPath != null) { - cmds.add(cmd); - } - } - compileCommands = cmds.toArray(new String[compileCommands.length]); - } - return compileCommands; + return new String[] { "gcc", "g++", "clang", "clang++", "cc", "c++" }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ //$NON-NLS-6$ } @Override - public IResource[] getResourcesFromCommand(String[] cmd, URI buildDirectoryURI) { + public String[] getCompileCommands(ILanguage language) { + if (GPPLanguage.ID.equals(language.getId())) { + return new String[] { "g++", "clang++", "c++" }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } else if (GCCLanguage.ID.equals(language.getId())) { + return new String[] { "gcc", "clang", "cc" }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } else { + return new String[0]; + } + } + + @Override + public IResource[] getResourcesFromCommand(List cmd, URI buildDirectoryURI) { // Start at the back looking for arguments List resources = new ArrayList<>(); IWorkspaceRoot root = ResourcesPlugin.getWorkspace().getRoot(); - for (int i = cmd.length - 1; i >= 0; --i) { - String arg = cmd[i]; + for (int i = cmd.size() - 1; i >= 0; --i) { + String arg = cmd.get(i); if (arg.startsWith("-")) { //$NON-NLS-1$ // ran into an option, we're done. break; } - for (IFile resource : root.findFilesForLocationURI(buildDirectoryURI.resolve(arg))) { + Path srcPath = Paths.get(arg); + URI uri; + if (srcPath.isAbsolute()) { + uri = srcPath.toUri(); + } else { + uri = buildDirectoryURI.resolve(arg); + } + + for (IFile resource : root.findFilesForLocationURI(uri)) { resources.add(resource); } } @@ -364,4 +433,22 @@ public class GCCToolChain extends PlatformObject implements IToolChain { return resources.toArray(new IResource[resources.size()]); } + @Override + public List stripCommand(List command, IResource[] resources) { + List newCommand = new ArrayList<>(); + + for (int i = 0; i < command.size() - resources.length; ++i) { + String arg = command.get(i); + if (arg.startsWith("-o")) { //$NON-NLS-1$ + if (arg.equals("-o")) { //$NON-NLS-1$ + i++; + } + continue; + } + newCommand.add(arg); + } + + return newCommand; + } + } diff --git a/build/org.eclipse.cdt.cmake.core/META-INF/MANIFEST.MF b/build/org.eclipse.cdt.cmake.core/META-INF/MANIFEST.MF index e7db1577fc6..2cf8671247f 100644 --- a/build/org.eclipse.cdt.cmake.core/META-INF/MANIFEST.MF +++ b/build/org.eclipse.cdt.cmake.core/META-INF/MANIFEST.MF @@ -10,7 +10,8 @@ Require-Bundle: org.eclipse.core.runtime, org.eclipse.debug.core;bundle-version="3.10.0", org.eclipse.launchbar.core;bundle-version="2.0.0", org.eclipse.cdt.core;bundle-version="5.12.0", - org.eclipse.tools.templates.freemarker;bundle-version="1.0.0";visibility:=reexport + org.eclipse.tools.templates.freemarker;bundle-version="1.0.0";visibility:=reexport, + com.google.gson Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Bundle-ActivationPolicy: lazy Export-Package: org.eclipse.cdt.cmake.core diff --git a/build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/CMakeProjectGenerator.java b/build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/CMakeProjectGenerator.java index 7b348fd28c7..b12a8a694da 100644 --- a/build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/CMakeProjectGenerator.java +++ b/build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/CMakeProjectGenerator.java @@ -55,13 +55,18 @@ public class CMakeProjectGenerator extends FMProjectGenerator { // Create the source folders IProject project = getProject(); List entries = new ArrayList<>(); - for (SourceRoot srcRoot : getManifest().getSrcRoots()) { - IFolder sourceFolder = project.getFolder(srcRoot.getDir()); - if (!sourceFolder.exists()) { - sourceFolder.create(true, true, monitor); + List srcRoots = getManifest().getSrcRoots(); + if (srcRoots != null && !srcRoots.isEmpty()) { + for (SourceRoot srcRoot : srcRoots) { + IFolder sourceFolder = project.getFolder(srcRoot.getDir()); + if (!sourceFolder.exists()) { + sourceFolder.create(true, true, monitor); + } + + entries.add(CoreModel.newSourceEntry(sourceFolder.getFullPath())); } - - entries.add(CoreModel.newSourceEntry(sourceFolder.getFullPath())); + } else { + entries.add(CoreModel.newSourceEntry(getProject().getFullPath())); } CoreModel.getDefault().create(project).setRawPathEntries(entries.toArray(new IPathEntry[entries.size()]), monitor); diff --git a/build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/internal/CMakeBuildConfiguration.java b/build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/internal/CMakeBuildConfiguration.java index 826c7521751..a1a4f7b2e92 100644 --- a/build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/internal/CMakeBuildConfiguration.java +++ b/build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/internal/CMakeBuildConfiguration.java @@ -8,6 +8,7 @@ package org.eclipse.cdt.cmake.core.internal; import java.io.File; +import java.io.FileReader; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -21,7 +22,6 @@ import org.eclipse.cdt.core.IConsoleParser; import org.eclipse.cdt.core.build.CBuildConfiguration; import org.eclipse.cdt.core.build.IToolChain; import org.eclipse.cdt.core.model.ICModelMarker; -import org.eclipse.cdt.core.parser.IScannerInfo; import org.eclipse.cdt.core.resources.IConsole; import org.eclipse.core.resources.IBuildConfiguration; import org.eclipse.core.resources.IProject; @@ -29,6 +29,8 @@ import org.eclipse.core.resources.IResource; import org.eclipse.core.runtime.CoreException; import org.eclipse.core.runtime.IProgressMonitor; +import com.google.gson.Gson; + public class CMakeBuildConfiguration extends CBuildConfiguration { public CMakeBuildConfiguration(IBuildConfiguration config, String name) throws CoreException { @@ -44,10 +46,10 @@ public class CMakeBuildConfiguration extends CBuildConfiguration { throws CoreException { IProject project = getProject(); try { - project.deleteMarkers(ICModelMarker.C_MODEL_PROBLEM_MARKER, false, IResource.DEPTH_INFINITE); + project.deleteMarkers(ICModelMarker.C_MODEL_PROBLEM_MARKER, false, IResource.DEPTH_INFINITE); ConsoleOutputStream outStream = console.getOutputStream(); - + Path buildDir = getBuildDirectory(); if (!Files.exists(buildDir.resolve("Makefile"))) { //$NON-NLS-1$ @@ -72,12 +74,16 @@ public class CMakeBuildConfiguration extends CBuildConfiguration { } project.refreshLocal(IResource.DEPTH_INFINITE, monitor); + + // Load compile_commands.json file + processCompileCommandsFile(monitor); + return new IProject[] { project }; } catch (IOException e) { throw new CoreException(Activator.errorStatus(String.format("Building %s", project.getName()), e)); } } - + @Override public void clean(IConsole console, IProgressMonitor monitor) throws CoreException { IProject project = getProject(); @@ -89,7 +95,7 @@ public class CMakeBuildConfiguration extends CBuildConfiguration { Path buildDir = getBuildDirectory(); if (!Files.exists(buildDir.resolve("Makefile"))) { //$NON-NLS-1$ - outStream.write("Makefile not found. Assuming clean"); + outStream.write("Makefile not found. Assuming clean."); return; } @@ -99,8 +105,6 @@ public class CMakeBuildConfiguration extends CBuildConfiguration { ProcessBuilder processBuilder = new ProcessBuilder(command).directory(buildDir.toFile()); Process process = processBuilder.start(); outStream.write(String.join(" ", command) + '\n'); //$NON-NLS-1$ - - // TODO error parsers watchProcess(process, new IConsoleParser[0], console); project.refreshLocal(IResource.DEPTH_INFINITE, monitor); @@ -109,10 +113,23 @@ public class CMakeBuildConfiguration extends CBuildConfiguration { } } - @Override - public IScannerInfo getScannerInformation(IResource resource) { - // TODO Auto-generated method stub - return null; + private void processCompileCommandsFile(IProgressMonitor monitor) throws CoreException { + IProject project = getProject(); + Path commandsFile = getBuildDirectory().resolve("compile_commands.json"); //$NON-NLS-1$ + if (Files.exists(commandsFile)) { + monitor.setTaskName("Processing compile_commands.json"); + try (FileReader reader = new FileReader(commandsFile.toFile())) { + Gson gson = new Gson(); + CompileCommand[] commands = gson.fromJson(reader, CompileCommand[].class); + for (CompileCommand command : commands) { + processLine(command.getCommand()); + } + shutdown(); + } catch (IOException e) { + throw new CoreException( + Activator.errorStatus(String.format("Processing compile commands %s", project.getName()), e)); + } + } } - + } diff --git a/build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/internal/CompileCommand.java b/build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/internal/CompileCommand.java new file mode 100644 index 00000000000..b672e67352d --- /dev/null +++ b/build/org.eclipse.cdt.cmake.core/src/org/eclipse/cdt/cmake/core/internal/CompileCommand.java @@ -0,0 +1,28 @@ +/******************************************************************************* + * Copyright (c) 2016 QNX Software Systems and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + *******************************************************************************/ +package org.eclipse.cdt.cmake.core.internal; + +public class CompileCommand { + + private String directory; + private String command; + private String file; + + public String getDirectory() { + return directory; + } + + public String getCommand() { + return command; + } + + public String getFile() { + return file; + } + +} diff --git a/build/org.eclipse.cdt.cmake.core/templates/simple/CMakeLists.txt b/build/org.eclipse.cdt.cmake.core/templates/simple/CMakeLists.txt index 010d47aa33a..6ee2edbbdc6 100644 --- a/build/org.eclipse.cdt.cmake.core/templates/simple/CMakeLists.txt +++ b/build/org.eclipse.cdt.cmake.core/templates/simple/CMakeLists.txt @@ -2,4 +2,4 @@ cmake_minimum_required (VERSION 2.6) project (${projectName}) -add_executable(${projectName} src/${projectName}.cpp) +add_executable(${projectName} ${projectName}.cpp) diff --git a/build/org.eclipse.cdt.cmake.core/templates/simple/manifest.xml b/build/org.eclipse.cdt.cmake.core/templates/simple/manifest.xml index fb88868dd9b..708c005910b 100644 --- a/build/org.eclipse.cdt.cmake.core/templates/simple/manifest.xml +++ b/build/org.eclipse.cdt.cmake.core/templates/simple/manifest.xml @@ -1,8 +1,7 @@ - \ No newline at end of file diff --git a/core/org.eclipse.cdt.core/META-INF/MANIFEST.MF b/core/org.eclipse.cdt.core/META-INF/MANIFEST.MF index 7c493205764..da5bb046840 100644 --- a/core/org.eclipse.cdt.core/META-INF/MANIFEST.MF +++ b/core/org.eclipse.cdt.core/META-INF/MANIFEST.MF @@ -127,6 +127,7 @@ Require-Bundle: org.eclipse.cdt.core.native;bundle-version="[5.7.0,6.0.0)";visib org.eclipse.core.variables;bundle-version="[3.1.100,4.0.0)", org.eclipse.ltk.core.refactoring;bundle-version="3.4.0", org.eclipse.text;bundle-version="[3.2.0,4.0.0)", - com.ibm.icu;bundle-version="4.4.2" + com.ibm.icu;bundle-version="4.4.2", + com.google.gson Bundle-ActivationPolicy: lazy Bundle-RequiredExecutionEnvironment: JavaSE-1.8 diff --git a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/CBuildConfiguration.java b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/CBuildConfiguration.java index a8b89aa6112..aa52d7812e8 100644 --- a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/CBuildConfiguration.java +++ b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/CBuildConfiguration.java @@ -9,18 +9,24 @@ package org.eclipse.cdt.core.build; import java.io.BufferedReader; import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintStream; +import java.lang.reflect.Type; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -30,14 +36,19 @@ import org.eclipse.cdt.core.IMarkerGenerator; import org.eclipse.cdt.core.ProblemMarkerInfo; import org.eclipse.cdt.core.envvar.IEnvironmentVariable; import org.eclipse.cdt.core.model.CoreModel; +import org.eclipse.cdt.core.model.ICElement; import org.eclipse.cdt.core.model.ICModelMarker; import org.eclipse.cdt.core.model.ICProject; import org.eclipse.cdt.core.model.IOutputEntry; import org.eclipse.cdt.core.model.IPathEntry; +import org.eclipse.cdt.core.model.ITranslationUnit; +import org.eclipse.cdt.core.parser.ExtendedScannerInfo; import org.eclipse.cdt.core.parser.IExtendedScannerInfo; import org.eclipse.cdt.core.parser.IScannerInfo; import org.eclipse.cdt.core.parser.IScannerInfoChangeListener; +import org.eclipse.cdt.core.parser.IncludeExportPatterns; import org.eclipse.cdt.core.resources.IConsole; +import org.eclipse.cdt.internal.core.parser.ParserSettings2; import org.eclipse.core.filesystem.URIUtil; import org.eclipse.core.resources.IBuildConfiguration; import org.eclipse.core.resources.IContainer; @@ -58,6 +69,15 @@ import org.eclipse.osgi.util.NLS; import org.osgi.service.prefs.BackingStoreException; import org.osgi.service.prefs.Preferences; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; + /** * Root class for CDT build configurations. Provides access to the build * settings for subclasses. @@ -72,10 +92,15 @@ public abstract class CBuildConfiguration extends PlatformObject private static final String TOOLCHAIN_ID = "cdt.toolChain.id"; //$NON-NLS-1$ private static final String TOOLCHAIN_VERSION = "cdt.toolChain.version"; //$NON-NLS-1$ + private static final List DEFAULT_COMMAND = new ArrayList<>(0); + private final String name; private final IBuildConfiguration config; private final IToolChain toolChain; + private final Map> scannerInfoListeners = new HashMap<>(); + private ScannerInfoCache scannerInfoCache; + protected CBuildConfiguration(IBuildConfiguration config, String name) throws CoreException { this.config = config; this.name = name; @@ -98,10 +123,9 @@ public abstract class CBuildConfiguration extends PlatformObject String.format("Toolchain missing for config: %s", config.getName()))); } } - toolChain = tc; } - + protected CBuildConfiguration(IBuildConfiguration config, String name, IToolChain toolChain) { this.config = config; this.name = name; @@ -121,7 +145,7 @@ public abstract class CBuildConfiguration extends PlatformObject protected CBuildConfiguration(IBuildConfiguration config, IToolChain toolChain) { this(config, DEFAULT_NAME, toolChain); } - + @Override public IBuildConfiguration getBuildConfiguration() { return config; @@ -374,56 +398,182 @@ public abstract class CBuildConfiguration extends PlatformObject } - private Map cheaterInfo; - private boolean infoChanged = false; + private File getScannerInfoCacheFile() { + return CCorePlugin.getDefault().getStateLocation().append("infoCache") //$NON-NLS-1$ + .append(getProject().getName()).append(name + ".json").toFile(); //$NON-NLS-1$ + } - private void initScannerInfo() { - if (cheaterInfo == null) { - cheaterInfo = new HashMap<>(); + private static class IExtendedScannerInfoCreator implements JsonDeserializer { + @Override + public IExtendedScannerInfo deserialize(JsonElement element, Type arg1, + JsonDeserializationContext arg2) throws JsonParseException { + JsonObject infoObj = element.getAsJsonObject(); + + Map definedSymbols = null; + if (infoObj.has("definedSymbols")) { //$NON-NLS-1$ + JsonObject definedSymbolsObj = infoObj.get("definedSymbols").getAsJsonObject(); //$NON-NLS-1$ + definedSymbols = new HashMap<>(); + for (Entry entry : definedSymbolsObj.entrySet()) { + definedSymbols.put(entry.getKey(), entry.getValue().getAsString()); + } + } + + String[] includePaths = null; + if (infoObj.has("includePaths")) { //$NON-NLS-1$ + JsonArray includePathsArray = infoObj.get("includePaths").getAsJsonArray(); //$NON-NLS-1$ + List includePathsList = new ArrayList<>(includePathsArray.size()); + for (Iterator i = includePathsArray.iterator(); i.hasNext();) { + includePathsList.add(i.next().getAsString()); + } + includePaths = includePathsList.toArray(new String[includePathsList.size()]); + } + + IncludeExportPatterns includeExportPatterns = null; + if (infoObj.has("includeExportPatterns")) { //$NON-NLS-1$ + JsonObject includeExportPatternsObj = infoObj.get("includeExportPatterns").getAsJsonObject(); //$NON-NLS-1$ + String exportPattern = null; + if (includeExportPatternsObj.has("includeExportPattern")) { //$NON-NLS-1$ + exportPattern = includeExportPatternsObj.get("includeExportPattern") //$NON-NLS-1$ + .getAsJsonObject().get("pattern").getAsString(); //$NON-NLS-1$ + } + + String beginExportsPattern = null; + if (includeExportPatternsObj.has("includeBeginExportPattern")) { //$NON-NLS-1$ + beginExportsPattern = includeExportPatternsObj.get("includeBeginExportPattern") //$NON-NLS-1$ + .getAsJsonObject().get("pattern").getAsString(); //$NON-NLS-1$ + } + + String endExportsPattern = null; + if (includeExportPatternsObj.has("includeEndExportPattern")) { //$NON-NLS-1$ + endExportsPattern = includeExportPatternsObj.get("includeEndExportPattern") //$NON-NLS-1$ + .getAsJsonObject().get("pattern").getAsString(); //$NON-NLS-1$ + } + + includeExportPatterns = new IncludeExportPatterns(exportPattern, beginExportsPattern, + endExportsPattern); + } + + ExtendedScannerInfo info = new ExtendedScannerInfo(definedSymbols, includePaths); + info.setIncludeExportPatterns(includeExportPatterns); + info.setParserSettings(new ParserSettings2()); + return info; } } + /** + * @since 6.1 + */ + protected void loadScannerInfoCache() { + if (scannerInfoCache == null) { + File cacheFile = getScannerInfoCacheFile(); + if (cacheFile.exists()) { + try (FileReader reader = new FileReader(cacheFile)) { + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.registerTypeAdapter(IExtendedScannerInfo.class, + new IExtendedScannerInfoCreator()); + Gson gson = gsonBuilder.create(); + scannerInfoCache = gson.fromJson(reader, ScannerInfoCache.class); + } catch (IOException e) { + CCorePlugin.log(e); + scannerInfoCache = new ScannerInfoCache(); + } + } else { + scannerInfoCache = new ScannerInfoCache(); + } + scannerInfoCache.initCache(); + } + } + + /** + * @since 6.1 + */ + protected void saveScannerInfoCache() { + File cacheFile = getScannerInfoCacheFile(); + if (!cacheFile.getParentFile().exists()) { + try { + Files.createDirectories(cacheFile.getParentFile().toPath()); + } catch (IOException e) { + CCorePlugin.log(e); + return; + } + } + + try (FileWriter writer = new FileWriter(getScannerInfoCacheFile())) { + Gson gson = new Gson(); + gson.toJson(scannerInfoCache, writer); + } catch (IOException e) { + CCorePlugin.log(e); + } + } + + /** + * @since 6.1 + */ + protected ScannerInfoCache getScannerInfoCache() { + return scannerInfoCache; + } + @Override public IScannerInfo getScannerInformation(IResource resource) { - initScannerInfo(); - return cheaterInfo.get(resource); + loadScannerInfoCache(); + IExtendedScannerInfo info = scannerInfoCache.getScannerInfo(resource); + if (info == null) { + ICElement celement = CCorePlugin.getDefault().getCoreModel().create(resource); + if (celement instanceof ITranslationUnit) { + ITranslationUnit tu = (ITranslationUnit) celement; + try { + info = getToolChain().getDefaultScannerInfo(getBuildConfiguration(), null, + tu.getLanguage(), getBuildDirectoryURI()); + scannerInfoCache.addScannerInfo(DEFAULT_COMMAND, info, resource); + saveScannerInfoCache(); + } catch (CoreException e) { + CCorePlugin.log(e.getStatus()); + } + } + } + return info; } + private boolean infoChanged = false; + @Override public boolean processLine(String line) { // TODO smarter line parsing to deal with quoted arguments - String[] command = line.split("\\s+"); //$NON-NLS-1$ + List command = Arrays.asList(line.split("\\s+")); //$NON-NLS-1$ // Make sure it's a compile command - boolean found = false; String[] compileCommands = toolChain.getCompileCommands(); + loop: for (String arg : command) { if (arg.startsWith("-")) { //$NON-NLS-1$ // option found, missed our command - break; + return false; } for (String cc : compileCommands) { - if (arg.equals(cc)) { - found = true; - break; + if (arg.endsWith(cc) + && (arg.equals(cc) || arg.endsWith("/" + cc) || arg.endsWith("\\" + cc))) { //$NON-NLS-1$ //$NON-NLS-2$ + break loop; } } } - if (!found) { - return false; - } - try { IResource[] resources = toolChain.getResourcesFromCommand(command, getBuildDirectoryURI()); if (resources != null) { + List commandStrings = toolChain.stripCommand(command, resources); + for (IResource resource : resources) { - initScannerInfo(); - cheaterInfo.put(resource, - getToolChain().getScannerInfo(getBuildConfiguration(), findCommand(command[0]), - Arrays.copyOfRange(command, 1, command.length), null, resource, - getBuildDirectoryURI())); + loadScannerInfoCache(); + if (scannerInfoCache.hasCommand(commandStrings)) { + scannerInfoCache.addResource(commandStrings, resource); + } else { + Path commandPath = findCommand(command.get(0)); + command.set(0, commandPath.toString()); + IExtendedScannerInfo info = getToolChain().getScannerInfo(getBuildConfiguration(), + command, null, resource, getBuildDirectoryURI()); + scannerInfoCache.addScannerInfo(commandStrings, info, resource); + } infoChanged = true; } return true; @@ -441,7 +591,9 @@ public abstract class CBuildConfiguration extends PlatformObject // TODO persist changes // Trigger a reindex if anything changed + // TODO be more surgical if (infoChanged) { + saveScannerInfoCache(); CCorePlugin.getIndexManager().reindex(CoreModel.getDefault().create(getProject())); infoChanged = false; } @@ -449,12 +601,23 @@ public abstract class CBuildConfiguration extends PlatformObject @Override public void subscribe(IResource resource, IScannerInfoChangeListener listener) { - // TODO for IScannerInfoProvider + List listeners = scannerInfoListeners.get(resource); + if (listeners == null) { + listeners = new ArrayList<>(); + scannerInfoListeners.put(resource, listeners); + } + listeners.add(listener); } @Override public void unsubscribe(IResource resource, IScannerInfoChangeListener listener) { - // TODO for IScannerInfoProvider + List listeners = scannerInfoListeners.get(resource); + if (listeners != null) { + listeners.remove(listener); + if (listeners.isEmpty()) { + scannerInfoListeners.remove(resource); + } + } } } diff --git a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/IToolChain.java b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/IToolChain.java index 66dee546127..22e90093fce 100644 --- a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/IToolChain.java +++ b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/IToolChain.java @@ -9,8 +9,12 @@ package org.eclipse.cdt.core.build; import java.net.URI; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.eclipse.cdt.core.envvar.IEnvironmentVariable; +import org.eclipse.cdt.core.model.ILanguage; import org.eclipse.cdt.core.parser.IExtendedScannerInfo; import org.eclipse.core.resources.IBuildConfiguration; import org.eclipse.core.resources.IResource; @@ -24,17 +28,48 @@ import org.eclipse.core.runtime.IAdaptable; */ public interface IToolChain extends IAdaptable { - // Standard attributes + /** + * Property: The OS the toolchain builds for. + */ static final String ATTR_OS = "os"; //$NON-NLS-1$ + + /** + * Property: The CPU architecture the toolchain supports. + */ static final String ATTR_ARCH = "arch"; //$NON-NLS-1$ + + /** + * Property: A package ID to reflect different version/package line up of + * the platform this toolchain supports. + */ static final String ATTR_PACKAGE = "package"; //$NON-NLS-1$ + /** + * The provider of the toolchain. + * + * @return toolchain provider + */ IToolChainProvider getProvider(); + /** + * The ID of the toolchain + * + * @return toolchain ID + */ String getId(); + /** + * The version of the toolchain + * + * @return toolchain version + */ String getVersion(); + /** + * The user friendly name for the toolchain + * + * @return toolchain name + */ String getName(); /** @@ -48,23 +83,165 @@ public interface IToolChain extends IAdaptable { */ String getProperty(String key); + /** + * Set a property on the toolchain. + * + * @param key + * key of the property + * @param value + * value of the property + */ void setProperty(String key, String value); - IEnvironmentVariable getVariable(String name); - + /** + * Return the environment variables to be set when invoking the tools in the + * toolchain. + * + * @return environment variables + */ IEnvironmentVariable[] getVariables(); - IExtendedScannerInfo getScannerInfo(IBuildConfiguration buildConfig, Path command, String[] args, - IExtendedScannerInfo baseScannerInfo, IResource resource, URI buildDirectoryURI); + /** + * Return the environment variable of the given name used when invoking the + * toolchain. + * + * @param name + * environment variable name + * @return environment variable value + */ + IEnvironmentVariable getVariable(String name); + /** + * Returns the error parser IDs use to create error markers for builds with + * this toolchain. + * + * @return error parser IDs + */ String[] getErrorParserIds(); - Path getCommandPath(Path command); - - String[] getCompileCommands(); - - IResource[] getResourcesFromCommand(String[] command, URI buildDirectoryURI); - + /** + * Returns the IDs for the binary parsers that can parse the build output of + * the toolchain. + * + * @return binary parser IDs for this toolchain + */ String getBinaryParserId(); + /** + * Get the scanner info for a given build config, command, base scanner + * info, resource and build directory. + * + * @param buildConfig + * the build configuration this scanner info applies to + * @param command + * the compile command that is used to build the resource + * @param baseScannerInfo + * base scanner info that this scanner info extends/replaces + * @param resource + * the resource this scanner info applies to, usually a source + * file + * @param buildDirectoryURI + * where the build command is run to build this resource + * @return scanner info for this resource + * @since 6.1 + */ + default IExtendedScannerInfo getScannerInfo(IBuildConfiguration buildConfig, List command, + IExtendedScannerInfo baseScannerInfo, IResource resource, URI buildDirectoryURI) { + return null; + } + + @Deprecated + default IExtendedScannerInfo getScannerInfo(IBuildConfiguration buildConfig, Path command, String[] args, + IExtendedScannerInfo baseScannerInfo, IResource resource, URI buildDirectoryURI) { + List commandStrings = new ArrayList<>(args.length + 1); + commandStrings.add(command.toString()); + commandStrings.addAll(Arrays.asList(args)); + return getScannerInfo(buildConfig, commandStrings, baseScannerInfo, resource, buildDirectoryURI); + } + + /** + * Return the default scanner info for this toolchain. This is used before + * any build information is available to provide at least a minimal scanner + * info based on the compiler built-ins. + * + * @param buildConfig + * the build configuration this scanner info applies to + * @param baseScannerInfo + * base scanner info that this scanner info extends/replaces + * @param language + * the source language that selects the tool to provide scanner + * info for + * @param buildDirectoryURI + * the build directory that would be used to run commands + * @returns default scanner info for this language + * @since 6.1 + */ + default IExtendedScannerInfo getDefaultScannerInfo(IBuildConfiguration buildConfig, + IExtendedScannerInfo baseScannerInfo, ILanguage language, URI buildDirectoryURI) { + return null; + } + + /** + * Returns the absolute path of the tool represented by the command + * + * @param command + * the command as it usually appears on the command line + * @return the absolute path to the tool for the command + */ + Path getCommandPath(Path command); + + /** + * Returns the list of compiler tools. + * + * @return list of compiler tools + */ + String[] getCompileCommands(); + + /** + * Returns the list of compiler tools for a given language. + * + * @param language + * the language for the commands + * @return the compile commands for the language + * @since 6.1 + */ + default String[] getCompileCommands(ILanguage language) { + return new String[0]; + } + + /** + * Returns the list of resources referenced in a compile command. + * + * @param command + * the compile command + * @param buildDirectoryURI + * the directory the compile command runs in + * @return the list of resources referenced in the compile command + * @since 6.1 + */ + default IResource[] getResourcesFromCommand(List command, URI buildDirectoryURI) { + return new IResource[0]; + } + + @Deprecated + default IResource[] getResourcesFromCommand(String[] command, URI buildDirectoryURI) { + return getResourcesFromCommand(Arrays.asList(command), buildDirectoryURI); + } + + /** + * Strips the resources from the compile command. Use to produce the common + * parts of the command shared by a number of resources. + * + * @param command + * the original compile command + * @param resources + * the resources this command compiles for usually returned by + * getResourcesFromCommand() + * @return the stripped command + * @since 6.1 + */ + default List stripCommand(List command, IResource[] resources) { + return command; + } + } diff --git a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ScannerInfoCache.java b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ScannerInfoCache.java new file mode 100644 index 00000000000..0c3e94d1437 --- /dev/null +++ b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ScannerInfoCache.java @@ -0,0 +1,126 @@ +/******************************************************************************* + * Copyright (c) 2016 QNX Software Systems and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + *******************************************************************************/ +package org.eclipse.cdt.core.build; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.eclipse.cdt.core.parser.IExtendedScannerInfo; +import org.eclipse.core.resources.IResource; + +/** + * Scanner info for a given build configuration. + * + * @since 6.1 + */ +public class ScannerInfoCache { + + private static class Command { + public List command; + public IExtendedScannerInfo info; + public List resourcePaths; + } + + private List commands; + + private transient Map, Command> commandMap = new HashMap<>(); + private transient Map resourceMap = new HashMap<>(); + + /** + * Initialize the cache of scanner info. Call this after loading this info + * using Gson. + */ + public void initCache() { + if (commands == null) { + commands = new ArrayList<>(); + } + + for (Command command : commands) { + commandMap.put(command.command, command); + for (String resourcePath : command.resourcePaths) { + resourceMap.put(resourcePath, command); + } + } + } + + public IExtendedScannerInfo getScannerInfo(IResource resource) { + String resourcePath = resource.getLocation().toOSString(); + Command command = resourceMap.get(resourcePath); + return command != null ? command.info : null; + } + + public IExtendedScannerInfo getScannerInfo(List commandStrings) { + Command command = commandMap.get(commandStrings); + return command != null ? command.info : null; + } + + public boolean hasCommand(List commandStrings) { + return commandMap.get(commandStrings) != null; + } + + public void addScannerInfo(List commandStrings, IExtendedScannerInfo info, IResource resource) { + // Do I need to remove the resource from an existing command? + String resourcePath = resource.getLocation().toOSString(); + Command oldCommand = resourceMap.get(resourcePath); + if (oldCommand != null) { + if (oldCommand.command.equals(commandStrings)) { + // duplicate + return; + } else { + oldCommand.resourcePaths.remove(resourcePath); + if (oldCommand.resourcePaths.isEmpty()) { + // unused, remove + commandMap.remove(commandStrings); + commands.remove(oldCommand); + resourceMap.remove(resourcePath); + } + } + } + + Command command = commandMap.get(commandStrings); + if (command != null) { + command.info = info; + command.resourcePaths.add(resourcePath); + resourceMap.put(resourcePath, command); + } else { + command = new Command(); + command.command = commandStrings; + command.info = info; + command.resourcePaths = new ArrayList<>(); + command.resourcePaths.add(resourcePath); + commands.add(command); + commandMap.put(commandStrings, command); + resourceMap.put(resourcePath, command); + } + } + + public void addResource(List commandStrings, IResource resource) { + String resourcePath = resource.getLocation().toOSString(); + Command command = commandMap.get(commandStrings); + Command current = resourceMap.get(resourcePath); + if (current != null) { + if (!current.equals(command)) { + // remove from old command + current.resourcePaths.remove(resourcePath); + if (current.resourcePaths.isEmpty()) { + commands.remove(current); + commandMap.remove(current.command); + } + } else { + // we're already there + return; + } + } else { + command.resourcePaths.add(resource.getLocation().toOSString()); + resourceMap.put(resourcePath, command); + } + } + +} diff --git a/debug/org.eclipse.cdt.debug.application.product/debug.product b/debug/org.eclipse.cdt.debug.application.product/debug.product index ac338125288..fe8724b97ea 100644 --- a/debug/org.eclipse.cdt.debug.application.product/debug.product +++ b/debug/org.eclipse.cdt.debug.application.product/debug.product @@ -177,6 +177,7 @@ Java and all Java-based trademarks are trademarks of Oracle Corporation in the U +