mirror of
https://github.com/eclipse-cdt/cdt
synced 2025-06-06 17:26:01 +02:00
Fixed up the automated integration suite to use the
new workbench test harness.
This commit is contained in:
Doug Schaefer
parent
a33d942f28
commit
53c3d3cbe1
3 changed files with 224 additions and 241 deletions
|
|
@ -235,5 +235,14 @@
|
|||
id="test.sub.sub">
|
||||
</target>
|
||||
</extension>
|
||||
<extension
|
||||
id="runTests"
|
||||
point="org.eclipse.core.runtime.applications">
|
||||
<application>
|
||||
<run
|
||||
class="org.eclipse.cdt.core.suite.RunTests">
|
||||
</run>
|
||||
</application>
|
||||
</extension>
|
||||
|
||||
</plugin>
|
||||
|
|
|
|||
|
|
@ -6,16 +6,8 @@
|
|||
*/
|
||||
package org.eclipse.cdt.core.suite;
|
||||
|
||||
import java.text.DecimalFormat;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import junit.framework.AssertionFailedError;
|
||||
import junit.framework.Test;
|
||||
import junit.framework.TestCase;
|
||||
import junit.framework.TestListener;
|
||||
import junit.framework.TestResult;
|
||||
import junit.framework.TestSuite;
|
||||
import junit.textui.TestRunner;
|
||||
|
||||
import org.eclipse.cdt.core.build.managed.tests.ManagedBuildTests;
|
||||
import org.eclipse.cdt.core.build.managed.tests.StandardBuildTests;
|
||||
|
|
@ -32,7 +24,6 @@ import org.eclipse.cdt.core.parser.failedTests.FailedCompleteParseASTTest;
|
|||
import org.eclipse.cdt.core.parser.failedTests.STLFailedTests;
|
||||
import org.eclipse.cdt.core.parser.tests.ParserTestSuite;
|
||||
import org.eclipse.cdt.core.search.tests.SearchTestSuite;
|
||||
import org.eclipse.core.boot.IPlatformRunnable;
|
||||
|
||||
/**
|
||||
* @author vhirsl
|
||||
|
|
@ -40,21 +31,7 @@ import org.eclipse.core.boot.IPlatformRunnable;
|
|||
* To change the template for this generated type comment go to
|
||||
* Window>Preferences>Java>Code Generation>Code and Comments
|
||||
*/
|
||||
public class AutomatedIntegrationSuite extends TestSuite
|
||||
implements TestListener, IPlatformRunnable {
|
||||
|
||||
private TestResult testResult = null;
|
||||
private String currentTestName;
|
||||
// success tests
|
||||
private int numberOfSuccessTests = 0;
|
||||
private int numberOfFailedSuccessTests = 0;
|
||||
// failed tests for open bug reports
|
||||
private int numberOfFailedTests = 0;
|
||||
private int numberOfFailedFailedTests = 0;
|
||||
// switching to failed tests
|
||||
protected boolean failedTests = false;
|
||||
protected boolean skipTest = false;
|
||||
|
||||
public class AutomatedIntegrationSuite extends TestSuite {
|
||||
|
||||
public AutomatedIntegrationSuite() {}
|
||||
|
||||
|
|
@ -73,9 +50,6 @@ public class AutomatedIntegrationSuite extends TestSuite
|
|||
public static Test suite() {
|
||||
final AutomatedIntegrationSuite suite = new AutomatedIntegrationSuite();
|
||||
|
||||
// First test to trigger report generation
|
||||
suite.addTest(suite.new GenerateReport("startSuccessTests"));
|
||||
|
||||
// Add all success tests
|
||||
suite.addTest(ManagedBuildTests.suite());
|
||||
suite.addTest(StandardBuildTests.suite());
|
||||
|
|
@ -91,7 +65,6 @@ public class AutomatedIntegrationSuite extends TestSuite
|
|||
//as the last test shuts down the indexing thread
|
||||
suite.addTest(IndexManagerTests.suite());
|
||||
// Last test to trigger report generation
|
||||
suite.addTest(suite.new GenerateReport("startFailedTests"));
|
||||
|
||||
// Add all failed tests
|
||||
suite.addTestSuite(ASTFailedTests.class);
|
||||
|
|
@ -99,220 +72,7 @@ public class AutomatedIntegrationSuite extends TestSuite
|
|||
suite.addTestSuite(CModelElementsFailedTests.class);
|
||||
suite.addTestSuite(FailedCompleteParseASTTest.class);
|
||||
|
||||
// Last test to trigger report generation
|
||||
suite.addTest(suite.new GenerateReport("generateReport"));
|
||||
|
||||
return suite;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the tests and collects their result in a TestResult.
|
||||
* Overloaded method
|
||||
*/
|
||||
public void run(TestResult result) {
|
||||
// To get counts from the result
|
||||
testResult = result;
|
||||
// Add oneself as a listener
|
||||
result.addListener(this);
|
||||
// Call a base class method
|
||||
super.run(result);
|
||||
// Remove a listener
|
||||
result.removeListener(this);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* An error occurred.
|
||||
*/
|
||||
public void addError(Test test, Throwable t) {
|
||||
// System.out.println("Error : " + test);
|
||||
// System.out.println("\tReason : " + t);
|
||||
// System.out.println("\tStack trace : ");
|
||||
// t.printStackTrace(System.out);
|
||||
}
|
||||
/**
|
||||
* A failure occurred.
|
||||
*/
|
||||
public void addFailure(Test test, AssertionFailedError t) {
|
||||
if (failedTests) {
|
||||
++numberOfFailedFailedTests;
|
||||
}
|
||||
else {
|
||||
++numberOfFailedSuccessTests;
|
||||
}
|
||||
// System.out.println("Failure : " + test);
|
||||
// System.out.println("\tReason : " + t);
|
||||
// System.out.println("\tStackTrace : ");
|
||||
// t.printStackTrace(System.out);
|
||||
}
|
||||
/**
|
||||
* A test ended.
|
||||
*/
|
||||
public void endTest(Test test) {
|
||||
if (currentTestName == null) {
|
||||
System.out.println("Internal error - endTest: currentTestName == null");
|
||||
}
|
||||
else {
|
||||
if (skipTest) {
|
||||
skipTest = false;
|
||||
}
|
||||
else {
|
||||
if (failedTests) {
|
||||
++numberOfFailedTests;
|
||||
// System.out.println(test);
|
||||
}
|
||||
else {
|
||||
++numberOfSuccessTests;
|
||||
}
|
||||
System.out.println(test);
|
||||
}
|
||||
currentTestName = null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* A test started.
|
||||
*/
|
||||
public void startTest(Test test) {
|
||||
if (currentTestName != null) {
|
||||
System.out.println("Internal error - startTest: currentTestName != null");
|
||||
}
|
||||
else {
|
||||
currentTestName = test.toString();
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* generateReport
|
||||
*
|
||||
* @author vhirsl
|
||||
*
|
||||
* To change the template for this generated type comment go to
|
||||
* Window>Preferences>Java>Code Generation>Code and Comments
|
||||
*/
|
||||
protected void generateReport() {
|
||||
int numberOfRuns = testResult.runCount();
|
||||
int numberOfFailures = testResult.failureCount();
|
||||
int numberOfErrors = testResult.errorCount();
|
||||
|
||||
System.out.println();
|
||||
System.out.println("*** Generating report: ***");
|
||||
System.out.println();
|
||||
System.out.println("\tNumber of runs: " + numberOfRuns);
|
||||
System.out.println("\tNumber of failures: " + numberOfFailures);
|
||||
System.out.println("\tNumber of errors: " + numberOfErrors);
|
||||
float successRate = (numberOfRuns-numberOfFailures-numberOfErrors)/(float)numberOfRuns;
|
||||
DecimalFormat df = new DecimalFormat("##.##%");
|
||||
System.out.println("Sanity success rate : " + df.format(successRate));
|
||||
System.out.println("\tNumber of success tests: " + numberOfSuccessTests);
|
||||
System.out.println("\tNumber of failed tests: " + numberOfFailedTests);
|
||||
successRate = numberOfSuccessTests/(float)(numberOfSuccessTests+numberOfFailedTests);
|
||||
System.out.println("Expected success test rate : " + df.format(successRate));
|
||||
successRate = (numberOfSuccessTests-numberOfFailedSuccessTests)/
|
||||
(float)(numberOfSuccessTests+numberOfFailedTests-numberOfFailedFailedTests);
|
||||
System.out.print("Observed success test rate : " + df.format(successRate));
|
||||
System.out.println(" (failed success tests = " + numberOfFailedSuccessTests + ", failed failed tests = " + numberOfFailedFailedTests + ")");
|
||||
System.out.println();
|
||||
}
|
||||
|
||||
protected void startSuccessTests() {
|
||||
failedTests = false;
|
||||
System.out.println();
|
||||
System.out.println("*** Starting success tests ***");
|
||||
System.out.println();
|
||||
}
|
||||
|
||||
protected void startFailedTests() {
|
||||
failedTests = true;
|
||||
System.out.println();
|
||||
System.out.println("*** Starting failed tests ***");
|
||||
System.out.println();
|
||||
}
|
||||
|
||||
/*
|
||||
* Public inner class to invoke generateReport
|
||||
*
|
||||
* @author vhirsl
|
||||
*
|
||||
* To change the template for this generated type comment go to
|
||||
* Window>Preferences>Java>Code Generation>Code and Comments
|
||||
*/
|
||||
public class GenerateReport extends TestCase {
|
||||
public GenerateReport(String name) {
|
||||
super(name);
|
||||
}
|
||||
public GenerateReport(){}
|
||||
|
||||
public void generateReport() {
|
||||
// skip this one
|
||||
AutomatedIntegrationSuite.this.skipTest = true;
|
||||
|
||||
// Calls a method of the outer class
|
||||
AutomatedIntegrationSuite.this.generateReport();
|
||||
}
|
||||
|
||||
public void startSuccessTests() {
|
||||
// skip this one
|
||||
AutomatedIntegrationSuite.this.skipTest = true;
|
||||
|
||||
// Calls a method of the outer class
|
||||
AutomatedIntegrationSuite.this.startSuccessTests();
|
||||
}
|
||||
|
||||
public void startFailedTests() {
|
||||
// skip this one
|
||||
AutomatedIntegrationSuite.this.skipTest = true;
|
||||
|
||||
// Calls a method of the outer class
|
||||
AutomatedIntegrationSuite.this.startFailedTests();
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see junit.framework.Test#countTestCases()
|
||||
* We don't want these test cases to be counted
|
||||
*/
|
||||
public int countTestCases() {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.eclipse.core.boot.IPlatformRunnable#run(java.lang.Object)
|
||||
*/
|
||||
public Object run(Object args) throws Exception {
|
||||
// Used when started from as a regression test suite after the build
|
||||
TestRunner testRunner = new TestRunner(new AISResultPrinter(System.out));
|
||||
TestResult testResult = testRunner.doRun(suite());
|
||||
|
||||
return prepareReport(testResult);
|
||||
}
|
||||
|
||||
protected ArrayList prepareReport(TestResult testResult) {
|
||||
// TestRunner.run(suite());
|
||||
ArrayList efMessages = new ArrayList();
|
||||
int errorCount = testResult.errorCount();
|
||||
int failureCount = testResult.failureCount();
|
||||
if (errorCount > 0) {
|
||||
String em = new String("There ");
|
||||
em += (errorCount == 1)?"is ":"are ";
|
||||
em += Integer.toString(errorCount);
|
||||
em += " error";
|
||||
em += (errorCount == 1)?"!":"s!";
|
||||
efMessages.add(em);
|
||||
}
|
||||
if (failureCount > 0) {
|
||||
String fm = new String("There ");
|
||||
fm += (failureCount == 1)?"is ":"are ";
|
||||
fm += Integer.toString(failureCount);
|
||||
fm += " failure";
|
||||
fm += (failureCount == 1)?"!":"s!";
|
||||
efMessages.add(fm);
|
||||
}
|
||||
if (efMessages.isEmpty()) {
|
||||
efMessages.add(new String("Regression test run SUCCESSFUL!"));
|
||||
}
|
||||
else {
|
||||
efMessages.add(new String("Please see raw test suite output for details."));
|
||||
}
|
||||
return efMessages;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,214 @@
|
|||
package org.eclipse.cdt.core.suite;
|
||||
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.PrintStream;
|
||||
|
||||
import junit.framework.AssertionFailedError;
|
||||
import junit.framework.Test;
|
||||
import junit.framework.TestListener;
|
||||
import junit.framework.TestResult;
|
||||
|
||||
import org.eclipse.core.boot.IPlatformRunnable;
|
||||
import org.eclipse.core.runtime.CoreException;
|
||||
import org.eclipse.core.runtime.IConfigurationElement;
|
||||
import org.eclipse.core.runtime.IExtension;
|
||||
import org.eclipse.core.runtime.Platform;
|
||||
import org.eclipse.ui.PlatformUI;
|
||||
import org.eclipse.ui.testing.ITestHarness;
|
||||
import org.eclipse.ui.testing.TestableObject;
|
||||
|
||||
/**
|
||||
* @see IPlatformRunnable
|
||||
*/
|
||||
public class RunTests implements IPlatformRunnable, ITestHarness, TestListener {
|
||||
|
||||
private TestableObject testableObject;
|
||||
private PrintStream stream;
|
||||
private String testReport;
|
||||
private String pluginName = "org.eclipse.cdt.core.tests";
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public RunTests() {
|
||||
}
|
||||
|
||||
/**
|
||||
* @see IPlatformRunnable#run
|
||||
*/
|
||||
public Object run(Object args) throws Exception {
|
||||
|
||||
stream = System.out;
|
||||
|
||||
String [] sargs = (String[])args;
|
||||
|
||||
for (int i = 0; i <sargs.length; ++i) {
|
||||
if (sargs[i].equals("-testout") && ++i < sargs.length) {
|
||||
stream = new PrintStream(new FileOutputStream(sargs[i]));
|
||||
} else if (sargs[i].equals("-testreport") && ++i < sargs.length) {
|
||||
testReport = sargs[i];
|
||||
}
|
||||
}
|
||||
|
||||
IPlatformRunnable application = getApplication();
|
||||
testableObject = PlatformUI.getTestableObject();
|
||||
testableObject.setTestHarness(this);
|
||||
return application.run(args);
|
||||
}
|
||||
|
||||
/*
|
||||
* return the application to run, or null if not even the default application
|
||||
* is found.
|
||||
*/
|
||||
private IPlatformRunnable getApplication() throws CoreException {
|
||||
// Assume we are in 3.0 mode.
|
||||
// Find the name of the application as specified by the PDE JUnit launcher.
|
||||
// If no application is specified, the 3.0 default workbench application
|
||||
// is returned.
|
||||
IExtension extension =
|
||||
Platform.getPluginRegistry().getExtension(
|
||||
Platform.PI_RUNTIME,
|
||||
Platform.PT_APPLICATIONS,
|
||||
"org.eclipse.ui.ide.workbench");
|
||||
|
||||
// If the extension does not have the correct grammar, return null.
|
||||
// Otherwise, return the application object.
|
||||
IConfigurationElement[] elements = extension.getConfigurationElements();
|
||||
if (elements.length > 0) {
|
||||
IConfigurationElement[] runs = elements[0].getChildren("run");
|
||||
if (runs.length > 0) {
|
||||
Object runnable = runs[0].createExecutableExtension("class");
|
||||
if (runnable instanceof IPlatformRunnable)
|
||||
return (IPlatformRunnable) runnable;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.eclipse.ui.testing.ITestHarness#runTests()
|
||||
*/
|
||||
public void runTests() {
|
||||
testableObject.testingStarting();
|
||||
testableObject.runTest(new Runnable() {
|
||||
public void run() {
|
||||
TestResult results = new TestResult();
|
||||
results.addListener(RunTests.this);
|
||||
|
||||
stream.println("<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>");
|
||||
stream.print("<?xml-stylesheet type=\"text/xsl\" href=\"");
|
||||
stream.print(testReport);
|
||||
stream.println("\"?>");
|
||||
|
||||
stream.print("<testRun name=\"");
|
||||
stream.print(pluginName);
|
||||
stream.println("\">");
|
||||
|
||||
startSuite(AutomatedIntegrationSuite.class.getName());
|
||||
AutomatedIntegrationSuite.suite().run(results);
|
||||
endSuite();
|
||||
|
||||
stream.print("</testRun>");
|
||||
results.removeListener(RunTests.this);
|
||||
results.stop();
|
||||
}
|
||||
});
|
||||
testableObject.testingFinished();
|
||||
}
|
||||
|
||||
// Test Listener methods
|
||||
|
||||
Throwable failure;
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see junit.framework.TestListener#addError(junit.framework.Test, java.lang.Throwable)
|
||||
*/
|
||||
public void addError(Test test, Throwable t) {
|
||||
failure = t;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see junit.framework.TestListener#addFailure(junit.framework.Test, junit.framework.AssertionFailedError)
|
||||
*/
|
||||
public void addFailure(Test test, AssertionFailedError t) {
|
||||
failure = t;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see junit.framework.TestListener#endTest(junit.framework.Test)
|
||||
*/
|
||||
public void endTest(Test test) {
|
||||
double time = (System.currentTimeMillis() - startTime) / 1000.0;
|
||||
|
||||
stream.print(" time=\"");
|
||||
stream.print(time);
|
||||
stream.print("\" result=\"");
|
||||
|
||||
boolean printerror = false;
|
||||
if (failure == null)
|
||||
stream.print("pass");
|
||||
else if (failure instanceof AssertionFailedError)
|
||||
stream.print("failed");
|
||||
else {
|
||||
stream.print("error");
|
||||
printerror = true;
|
||||
}
|
||||
|
||||
stream.print("\"");
|
||||
|
||||
if (printerror) {
|
||||
stream.println(">");
|
||||
failure.printStackTrace(stream);
|
||||
stream.println("\t\t\t</test>");
|
||||
} else
|
||||
stream.println("/>");
|
||||
|
||||
failure = null;
|
||||
}
|
||||
|
||||
private Class currentTest;
|
||||
|
||||
private long startTime;
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see junit.framework.TestListener#startTest(junit.framework.Test)
|
||||
*/
|
||||
public void startTest(Test test) {
|
||||
if (test.getClass() != currentTest) {
|
||||
if (currentTest != null)
|
||||
endClass();
|
||||
stream.print("\t\t<testClass name=\"");
|
||||
stream.print(test.getClass().getName());
|
||||
stream.println("\">");
|
||||
currentTest = test.getClass();
|
||||
}
|
||||
|
||||
String name = test.toString();
|
||||
name = name.substring(0, name.indexOf('('));
|
||||
stream.print("\t\t\t<test name=\"");
|
||||
stream.print(name);
|
||||
stream.print("\"");
|
||||
|
||||
startTime = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
// Report Generator
|
||||
|
||||
private void startSuite(String name) {
|
||||
stream.print("\t<testSuite name=\"");
|
||||
stream.print(name);
|
||||
stream.println("\">");
|
||||
}
|
||||
|
||||
private void endSuite() {
|
||||
if (currentTest != null) {
|
||||
endClass();
|
||||
currentTest = null;
|
||||
}
|
||||
stream.println("\t</testSuite>");
|
||||
}
|
||||
|
||||
private void endClass() {
|
||||
stream.println("\t\t</testClass>");
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue