PR 62809 [Scanner Config] Handling -DMACRO="value"

PR 63330 [Scanner Config] Auto-discovering dirs with escaped spaces
PR 64226 [Scanner Config] Path discovery supporting UNC filepaths

build/org.eclipse.cdt.make.core/src/org/eclipse/cdt/make/core/scannerconfig/ScannerConfigUtil.java

@@ -12,7 +12,6 @@ package org.eclipse.cdt.make.core.scannerconfig;
 
 import java.io.File;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
@@ -268,13 +267,18 @@ public final class ScannerConfigUtil {
 	 * @param String
 	 * @return String[] 
 	 */
-	public static String[] tokenizeStringWithQuotes(String line) {
+	public static String[] tokenizeStringWithQuotes(String line, String quoteStyle) {
 		ArrayList allTokens = new ArrayList();
-		String[] tokens = line.split("\"");	//$NON-NLS-1$
+		String[] tokens = line.split(quoteStyle);
 		for (int i = 0; i < tokens.length; ++i) {
 			if (i % 2 == 0) { // even tokens need further tokenization
 				String[] sTokens = tokens[i].split("\\s"); //$NON-NLS-1$
-				allTokens.addAll(Arrays.asList(sTokens));
+				// remove empty strings
+				for (int j = 0; j < sTokens.length; ++j) {
+					if (sTokens[j].length() > 0) {
+						allTokens.add(sTokens[j]);
+					}
+				}
 			}
 			else {
 				allTokens.add(tokens[i]);

build/org.eclipse.cdt.make.core/src/org/eclipse/cdt/make/internal/core/scannerconfig/DefaultExternalScannerInfoProvider.java

@@ -104,7 +104,7 @@ public class DefaultExternalScannerInfoProvider implements IExternalScannerInfoP
 			
 			OutputStream sniffer = ScannerInfoConsoleParserFactory.getESIProviderOutputSniffer(
 					cos, currentProject, buildInfo, collector);
-			TraceUtil.outputTrace("Default provider running command:", fCompileCommand.toString() + ca, ""); //$NON-NLS-1$ //$NON-NLS-2$
+			TraceUtil.outputTrace("Default provider is executing command:", fCompileCommand.toString() + ca, ""); //$NON-NLS-1$ //$NON-NLS-2$
 			Process p = launcher.execute(fCompileCommand, compileArguments, setEnvironment(launcher), fWorkingDirectory);
 			if (p != null) {
 				try {
@@ -168,7 +168,7 @@ public class DefaultExternalScannerInfoProvider implements IExternalScannerInfoP
 		}
 		fCompileCommand = buildInfo.getESIProviderCommand();
 		if (fCompileCommand != null) {
-			fCompileArguments = ScannerConfigUtil.tokenizeStringWithQuotes(buildInfo.getESIProviderArguments());
+			fCompileArguments = ScannerConfigUtil.tokenizeStringWithQuotes(buildInfo.getESIProviderArguments(), "\"");//$NON-NLS-1$
 			for (int i = 0; i < fCompileArguments.length; ++i) {
 				fCompileArguments[i] = fCompileArguments[i].replaceAll("\\$\\{plugin_state_location\\}",	//$NON-NLS-1$ 
 						MakeCorePlugin.getWorkingDirectory().toString());

build/org.eclipse.cdt.make.core/src/org/eclipse/cdt/make/internal/core/scannerconfig/gnu/GCCScannerInfoConsoleParser.java

@@ -16,7 +16,6 @@ import org.eclipse.cdt.core.IMarkerGenerator;
 import org.eclipse.cdt.make.core.scannerconfig.IScannerInfoCollector;
 import org.eclipse.cdt.make.core.scannerconfig.IScannerInfoConsoleParser;
 import org.eclipse.cdt.make.core.scannerconfig.IScannerInfoConsoleParserUtility;
-import org.eclipse.cdt.make.core.scannerconfig.ScannerConfigUtil;
 import org.eclipse.cdt.make.internal.core.MakeMessages;
 import org.eclipse.cdt.make.internal.core.scannerconfig.util.TraceUtil;
 
@@ -33,11 +32,16 @@ import java.util.Map;
  * @author vhirsl
  */
 public class GCCScannerInfoConsoleParser implements IScannerInfoConsoleParser {
+	private final static String SINGLE_QUOTE_STRING = "\'"; //$NON-NLS-1$
+	private final static String DOUBLE_QUOTE_STRING = "\""; //$NON-NLS-1$
 	
 	private IProject fProject = null;
 	private IScannerInfoConsoleParserUtility fUtil = null;
 	private IScannerInfoCollector fCollector = null;
 	
+	private boolean bMultiline = false;
+	private String sMultiline = ""; //$NON-NLS-1$
+	
 	/* (non-Javadoc)
 	 * @see org.eclipse.cdt.make.core.scannerconfig.IScannerInfoConsoleParser#startup(org.eclipse.core.resources.IProject, org.eclipse.cdt.make.internal.core.scannerconfig.IScannerInfoConsoleParserUtility, org.eclipse.cdt.make.core.scannerconfig.IScannerInfoCollector)
 	 */
@@ -52,6 +56,17 @@ public class GCCScannerInfoConsoleParser implements IScannerInfoConsoleParser {
 	 */
 	public boolean processLine(String line) {
 		boolean rc = false;
+		// check for multiline commands (ends with '\')
+		if (line.endsWith("\\")) { //$NON-NLS-1$
+			sMultiline += line.substring(0, line.length()-1);// + " "; //$NON-NLS-1$
+			bMultiline = true;
+			return rc;
+		}
+		if (bMultiline) {
+			line = sMultiline + line;
+			bMultiline = false;
+			sMultiline = ""; //$NON-NLS-1$
+		}
 		TraceUtil.outputTrace("GCCScannerInfoConsoleParser parsing line:", TraceUtil.EOL, line);	//$NON-NLS-1$ //$NON-NLS-2$
 		// make\[[0-9]*\]:  error_desc
 		int firstColon= line.indexOf(':');
@@ -65,19 +80,21 @@ public class GCCScannerInfoConsoleParser implements IScannerInfoConsoleParser {
 			    int e = msg.indexOf('\'');
 			    if (s != -1 && e != -1) {
 			    	String dir = msg.substring(s+1, e);
+					if (fUtil != null) {
 						fUtil.changeMakeDirectory(dir, getDirectoryLevel(line), enter);
+					}
 			    	return rc;
 				}
 			}
 		}
 		// Known patterns:
 		// (a) gcc|g++ ... -Dxxx -Iyyy ...
-		ArrayList allTokens = new ArrayList(Arrays.asList(ScannerConfigUtil.tokenizeStringWithQuotes(line)));
+		ArrayList allTokens = new ArrayList(Arrays.asList(line.split("\\s")));//$NON-NLS-1$
 		if (allTokens.size() <= 1)
 			return false;
 		Iterator I = allTokens.iterator();
 		String token = ((String) I.next()).toLowerCase();
-		if (token.indexOf("gcc") != -1 || token.indexOf("g++") != -1 || token.indexOf("qcc") != -1) {//$NON-NLS-1$ //$NON-NLS-2$
+		if (token.indexOf("gcc") != -1 || token.indexOf("g++") != -1 || token.indexOf("gcc") != -1) {//$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
 			// Recognized gcc or g++ compiler invocation
 			List includes = new ArrayList();
 			List symbols = new ArrayList();
@@ -85,54 +102,14 @@ public class GCCScannerInfoConsoleParser implements IScannerInfoConsoleParser {
 
 			rc = true;
 			String fileName = null;
-			String cashedToken = null;
+			// discover all -I options
+			parseLineForIncludePaths(line, includes);
+			// discover all -D options
+			parseLineForSymbolDefinitions(line, symbols);
+			
 			while (I.hasNext()) {
-				if (cashedToken == null) {
 				token = (String) I.next();
-				}
+				if (token.equals("-mwin32") ||		//$NON-NLS-1$
-				else {
-					token = cashedToken;
-					cashedToken = null;
-				}
-				if (token.length() == 0) {
-					continue;
-				}
-				if (token.startsWith("-D")) {//$NON-NLS-1$
-					String symbol = token.substring(2);
-					if (symbol.length() == 0) {
-						if (I.hasNext()) {
-							symbol = (String) I.next();
-						}
-						else {
-							continue;
-						}
-					}
-					if (symbol.charAt(0) == '-') {
-						cashedToken = symbol;
-						continue;
-					}
-					if (!symbols.contains(symbol))
-						symbols.add(symbol);
-				}
-				else if (token.startsWith("-I")) {//$NON-NLS-1$
-					String iPath = token.substring(2);
-					if (iPath.length() == 0) {
-						if (I.hasNext()) {
-							iPath = (String) I.next();
-						}
-						else {
-							continue;
-						}
-					}
-					if (iPath.charAt(0) == '-') {
-						cashedToken = iPath;
-						continue;
-					}
-					String nPath = fUtil.normalizePath(iPath);
-					if (!includes.contains(nPath))
-						includes.add(nPath);
-				}
-				else if (token.equals("-mwin32") ||		//$NON-NLS-1$
 					token.equals("-mno-win32") ||	//$NON-NLS-1$
 					token.equals("-mno-cygwin") ||	//$NON-NLS-1$
 					token.equals("-ansi") ||		//$NON-NLS-1$
@@ -164,18 +141,22 @@ public class GCCScannerInfoConsoleParser implements IScannerInfoConsoleParser {
 			List translatedIncludes = includes;
 			if (includes.size() > 0) {
 				if (fileName != null) {
+					if (fUtil != null) {
 						file = fUtil.findFile(fileName);
 						if (file != null) {
 							project = file.getProject();
 							translatedIncludes = fUtil.translateRelativePaths(file, fileName, includes);
 						}
 					}
+				}
 				else {
 					final String error = MakeMessages.getString("ConsoleParser.Filename_Missing_Error_Message"); //$NON-NLS-1$ 
 					TraceUtil.outputError(error, line);
+					if (fUtil != null) {
 						fUtil.generateMarker(fProject, -1, error + line, IMarkerGenerator.SEVERITY_WARNING, null);
 					}
-				if (file == null) {
+				}
+				if (file == null && fUtil != null) {	// real world case
 					// remove include paths since there was no chance to translate them
 					translatedIncludes.clear();
 				}
@@ -193,6 +174,124 @@ public class GCCScannerInfoConsoleParser implements IScannerInfoConsoleParser {
 		return rc;
 	}
 
+	/**
+	 * @param line
+	 * @param includes
+	 */
+	private void parseLineForIncludePaths(String line, List includes) {
+		final String fDashI = "-I"; //$NON-NLS-1$
+		int prevIndex = 0;
+		for (int index = line.indexOf(fDashI, prevIndex); index != -1; 
+			 prevIndex = index+2, index = line.indexOf(fDashI, prevIndex)) {
+			String delimiter = "\\s"; //$NON-NLS-1$
+			if (line.charAt(index-1) == '\'' || line.charAt(index-1) == '\"') {
+				// look for only one more ' or "
+				delimiter = String.valueOf(line.charAt(index-1));
+			}
+			String postfix = line.substring(index+2).trim();
+			if (postfix.charAt(0) == '-') {	// empty -I
+				continue;
+			}
+			if (postfix.startsWith(SINGLE_QUOTE_STRING) || postfix.startsWith(DOUBLE_QUOTE_STRING)) { //$NON-NLS-1$ //$NON-NLS-2$
+				delimiter = postfix.substring(0, 1);
+			}
+			String[] tokens = postfix.split(delimiter);
+			int tokIndex = (tokens.length > 1 && tokens[0].length() == 0) ? 1 : 0;
+			String iPath = tokens[tokIndex];
+			String temp = iPath;
+			// check for '\ '
+			for (++tokIndex; (temp.endsWith("\\") && tokIndex < tokens.length &&  //$NON-NLS-1$
+							 tokens[tokIndex].length() > 0 && !tokens[tokIndex].startsWith("-")); //$NON-NLS-1$
+				 ++tokIndex) { 
+				int beg = postfix.indexOf(temp)+temp.length();
+				int end = postfix.indexOf(tokens[tokIndex])+tokens[tokIndex].length();
+				iPath = iPath.substring(0, iPath.length()-1) + postfix.substring(beg, end);
+				temp += postfix.substring(beg, end);
+			}
+			String nPath = iPath;
+			if (fUtil != null) {
+				nPath = fUtil.normalizePath(iPath);
+			}
+			if (!includes.contains(nPath)) {
+				includes.add(nPath);
+			}
+		}
+	}
+
+	/**
+	 * @param line
+	 * @param symbols
+	 */
+	private void parseLineForSymbolDefinitions(String line, List symbols) {
+		final String fDashD = "-D"; //$NON-NLS-1$
+		int prevIndex = 0;
+		for (int index = line.indexOf(fDashD, prevIndex); index != -1; 
+			 prevIndex = index+2, index = line.indexOf(fDashD, prevIndex)) {
+			String delimiter = "\\s"; //$NON-NLS-1$
+			int nDelimiterSymbols = 2;
+			String postfix = line.substring(index+2).trim();
+			if (postfix.charAt(0) == '-') {	// empty -D
+				continue;
+			}
+			if (line.charAt(index-1) == '\'' || line.charAt(index-1) == '\"') {
+				// look for only one more ' or "
+				delimiter = String.valueOf(line.charAt(index-1)); 
+				nDelimiterSymbols = 1;
+			}
+			else {
+				String[] tokens = postfix.split(delimiter, 2);
+				if (tokens.length > 0 && tokens[0].length() > 0) {
+					int sQuoteIndex = tokens[0].indexOf(SINGLE_QUOTE_STRING);
+					int dQuoteIndex = tokens[0].indexOf(DOUBLE_QUOTE_STRING);
+					if (sQuoteIndex == -1 && dQuoteIndex == -1) {
+						// simple case, no quotes
+						if (!symbols.contains(tokens[0])) {
+							symbols.add(tokens[0]);
+						}
+					}
+					else {
+						delimiter = (sQuoteIndex != -1 && (dQuoteIndex == -1 || sQuoteIndex < dQuoteIndex)) ? SINGLE_QUOTE_STRING : DOUBLE_QUOTE_STRING;
+					}
+				}
+				else 
+					continue;
+			}
+			
+			// find next matching delimiter
+			int nextDelimiterIndex = -1;
+			int prevDelimiterIndex = -1;
+			do {
+				nextDelimiterIndex = postfix.indexOf(delimiter, nextDelimiterIndex+1);
+				if (nextDelimiterIndex == 0 || (nextDelimiterIndex > 0 && postfix.charAt(nextDelimiterIndex-1) != '\\')) {
+					--nDelimiterSymbols;
+					if (nDelimiterSymbols > 0) {
+						prevDelimiterIndex = nextDelimiterIndex;
+					}
+				}
+			}
+			while (nDelimiterSymbols > 0 && nextDelimiterIndex != -1);
+			if (nDelimiterSymbols > 0) 
+				continue; // non matching delimiter
+
+			// take everything up to the last delimiter
+			boolean bStartsWithDelimiter = postfix.startsWith(delimiter);
+			String symbol = postfix.substring(bStartsWithDelimiter ? 1 : 0, nextDelimiterIndex);
+			if (!bStartsWithDelimiter) {
+				// there is still a delimiter to be removed
+				if (prevDelimiterIndex != -1) {
+					symbol = symbol.substring(0, prevDelimiterIndex) + symbol.substring(prevDelimiterIndex+1);
+				}
+			}
+			// transform '\"' into '"'
+			if (delimiter.equals(DOUBLE_QUOTE_STRING)) {
+				symbol = symbol.replaceAll("\\\\\"", DOUBLE_QUOTE_STRING); //$NON-NLS-1$
+			}
+			if (!symbols.contains(symbol)) {
+				symbols.add(symbol);
+			}
+		}
+	}
+
 	/* (non-Javadoc)
 	 * @see org.eclipse.cdt.make.core.scannerconfig.IScannerInfoConsoleParser#shutdown()
 	 */