Skip to content

Commit

Permalink
Represent test timeouts in XML
Browse files Browse the repository at this point in the history 
This creates similar output as bazelbuild/bazel@7b091c1
  • Loading branch information
@illicitonion
illicitonion committed Jan 17, 2024
1 parent 306e4da commit cb7206c
Show file tree
Hide file tree
Showing 5 changed files with 124 additions and 40 deletions.
7 changes: 7 additions & 0 deletions java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/ActualRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,13 @@ public boolean run(String testClassName) {
}

try (BazelJUnitOutputListener bazelJUnitXml = new BazelJUnitOutputListener(xmlOut)) {
Runtime.getRuntime()
.addShutdownHook(
new Thread(
() -> {
bazelJUnitXml.closeForInterrupt();
}));

CommandLineSummary summary = new CommandLineSummary();
FailFastExtension failFastExtension = new FailFastExtension();

Expand Down
77 changes: 55 additions & 22 deletions java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOutputListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
Expand All @@ -30,9 +31,15 @@ public class BazelJUnitOutputListener implements TestExecutionListener, Closeabl
public static final Logger LOG = Logger.getLogger(BazelJUnitOutputListener.class.getName());
private final XMLStreamWriter xml;

private final Object resultsLock = new Object();
// Commented out to avoid adding a dependency to building the test runner.
// @GuardedBy("resultsLock")
private final Map<UniqueId, TestData> results = new ConcurrentHashMap<>();
private TestPlan testPlan;

private final AtomicBoolean hasClosed = new AtomicBoolean();
private final AtomicBoolean wasInterrupted = new AtomicBoolean();

public BazelJUnitOutputListener(Path xmlOut) {
try {
Files.createDirectories(xmlOut.getParent());
Expand Down Expand Up @@ -72,7 +79,8 @@ public void testPlanExecutionFinished(TestPlan testPlan) {
this.testPlan = null;
}

private Map<TestData, List<TestData>> matchTestCasesToSuites(List<TestData> testCases) {
// Requires the caller to have acquired resultsLock.
private Map<TestData, List<TestData>> matchTestCasesToSuites_locked(List<TestData> testCases) {
Map<TestData, List<TestData>> knownSuites = new HashMap<>();

// Find the containing test suites for the test cases.
Expand All @@ -91,7 +99,8 @@ private Map<TestData, List<TestData>> matchTestCasesToSuites(List<TestData> test
return knownSuites;
}

private List<TestData> findTestCases() {
// Requires the caller to have acquired resultsLock.
private List<TestData> findTestCases_locked() {
return results.values().stream()
// Ignore test plan roots. These are always the engine being used.
.filter(result -> !testPlan.getRoots().contains(result.getId()))
Expand Down Expand Up @@ -134,28 +143,39 @@ private void outputIfTestRootIsComplete(TestIdentifier testIdentifier) {
return;
}

List<TestData> testCases = findTestCases();
Map<TestData, List<TestData>> testSuites = matchTestCasesToSuites(testCases);
output(false);
}

// Write the results
try {
for (Map.Entry<TestData, List<TestData>> suiteAndTests : testSuites.entrySet()) {
new TestSuiteXmlRenderer(testPlan)
.toXml(xml, suiteAndTests.getKey(), suiteAndTests.getValue());
private void output(boolean isInterrupted) {
synchronized (this.resultsLock) {
List<TestData> testCases = findTestCases_locked();
Map<TestData, List<TestData>> testSuites = matchTestCasesToSuites_locked(testCases);

// Write the results
try {
for (Map.Entry<TestData, List<TestData>> suiteAndTests : testSuites.entrySet()) {
if (isInterrupted) {
for (TestData test : suiteAndTests.getValue()) {
test.mark(TestExecutionResult.aborted(null));
}
}
new TestSuiteXmlRenderer(testPlan)
.toXml(xml, suiteAndTests.getKey(), suiteAndTests.getValue(), isInterrupted);
}
} catch (XMLStreamException e) {
throw new RuntimeException(e);
}
} catch (XMLStreamException e) {
throw new RuntimeException(e);
}

// Delete the results we've used to conserve memory. This is safe to do
// since we only do this when the test root is complete, so we know that
// we won't be adding to the list of suites and test cases for that root
// (because tests and containers are arranged in a hierarchy --- the
// containers only complete when all the things they contain are
// finished. We are leaving all the test data that we have _not_ written
// to the XML file.
Stream.concat(testCases.stream(), testSuites.keySet().stream())
.forEach(data -> results.remove(data.getId().getUniqueIdObject()));
// Delete the results we've used to conserve memory. This is safe to do
// since we only do this when the test root is complete, so we know that
// we won't be adding to the list of suites and test cases for that root
// (because tests and containers are arranged in a hierarchy --- the
// containers only complete when all the things they contain are
// finished. We are leaving all the test data that we have _not_ written
// to the XML file.
Stream.concat(testCases.stream(), testSuites.keySet().stream())
.forEach(data -> results.remove(data.getId().getUniqueIdObject()));
}
}

@Override
Expand All @@ -164,10 +184,23 @@ public void reportingEntryPublished(TestIdentifier testIdentifier, ReportEntry e
}

private TestData getResult(TestIdentifier id) {
return results.computeIfAbsent(id.getUniqueIdObject(), ignored -> new TestData(id));
synchronized (resultsLock) {
return results.computeIfAbsent(id.getUniqueIdObject(), ignored -> new TestData(id));
}
}

public void closeForInterrupt() {
wasInterrupted.set(true);
close();
}

public void close() {
if (hasClosed.getAndSet(true)) {
return;
}
if (wasInterrupted.get()) {
output(true);
}
try {
xml.writeEndDocument();
xml.close();
Expand Down
31 changes: 19 additions & 12 deletions java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/TestCaseXmlRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ public TestCaseXmlRenderer(TestPlan testPlan) {
this.testPlan = testPlan;
}

public void toXml(XMLStreamWriter xml, TestData test) throws XMLStreamException {
public void toXml(XMLStreamWriter xml, TestData test, boolean isInterrupted)
throws XMLStreamException {
DecimalFormat decimalFormat = new DecimalFormat("#.##", DECIMAL_FORMAT_SYMBOLS);
decimalFormat.setRoundingMode(RoundingMode.HALF_UP);

Expand All @@ -49,19 +50,25 @@ public void toXml(XMLStreamWriter xml, TestData test) throws XMLStreamException
xml.writeAttribute("classname", LegacyReportingUtils.getClassName(testPlan, id));
xml.writeAttribute("time", decimalFormat.format(test.getDuration().toMillis() / 1000f));

if (test.isDisabled() || test.isSkipped()) {
xml.writeStartElement("skipped");
if (test.getSkipReason() != null) {
xml.writeCData(test.getSkipReason());
} else {
if (isInterrupted) {
xml.writeStartElement("failure");
xml.writeCData("Test timed out and was interrupted");
xml.writeEndElement();
} else {
if (test.isDisabled() || test.isSkipped()) {
xml.writeStartElement("skipped");
if (test.getSkipReason() != null) {
xml.writeCData(test.getSkipReason());
} else {
writeThrowableMessage(xml, test.getResult());
}
xml.writeEndElement();
}
if (test.isFailure() || test.isError()) {
xml.writeStartElement(test.isFailure() ? "failure" : "error");
writeThrowableMessage(xml, test.getResult());
xml.writeEndElement();
}
xml.writeEndElement();
}
if (test.isFailure() || test.isError()) {
xml.writeStartElement(test.isFailure() ? "failure" : "error");
writeThrowableMessage(xml, test.getResult());
xml.writeEndElement();
}

writeTextElement(xml, "system-out", test.getStdOut());
Expand Down
9 changes: 7 additions & 2 deletions java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/TestSuiteXmlRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ public TestSuiteXmlRenderer(TestPlan testPlan) {
testRenderer = new TestCaseXmlRenderer(testPlan);
}

public void toXml(XMLStreamWriter xml, TestData suite, Collection<TestData> tests)
public void toXml(
XMLStreamWriter xml, TestData suite, Collection<TestData> tests, boolean isInterrupted)
throws XMLStreamException {
xml.writeStartElement("testsuite");

Expand All @@ -27,6 +28,10 @@ public void toXml(XMLStreamWriter xml, TestData suite, Collection<TestData> test
int disabled = 0;
int skipped = 0;
for (TestData result : tests) {
if (isInterrupted) {
failures++;
continue;
}
if (result.isError()) {
errors++;
}
Expand All @@ -51,7 +56,7 @@ public void toXml(XMLStreamWriter xml, TestData suite, Collection<TestData> test
xml.writeEmptyElement("properties");

for (TestData testCase : tests) {
testRenderer.toXml(xml, testCase);
testRenderer.toXml(xml, testCase, isInterrupted);
}

writeTextElement(xml, "system-out", suite.getStdOut());
Expand Down
40 changes: 36 additions & 4 deletions java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOuputListenerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.platform.launcher.LauncherConstants.STDERR_REPORT_ENTRY_KEY;
import static org.junit.platform.launcher.LauncherConstants.STDOUT_REPORT_ENTRY_KEY;
import static org.mockito.Mockito.when;

import java.io.IOException;
import java.io.Reader;
Expand All @@ -15,6 +16,7 @@
import java.io.Writer;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
Expand Down Expand Up @@ -124,7 +126,7 @@ public void disabledTestsAreMarkedAsSkipped() {
new TestData(childId)
.mark(TestExecutionResult.aborted(new TestAbortedException("skipping is fun")));

Document xml = generateTestXml(testPlan, suite, List.of(childResult));
Document xml = generateTestXml(testPlan, suite, List.of(childResult), false);

// Because of the way we generated the XML, the root element is the `testsuite` one
Element root = xml.getDocumentElement();
Expand All @@ -141,6 +143,35 @@ public void disabledTestsAreMarkedAsSkipped() {
assertNotNull(skipped);
}

@Test
public void interruptedTestsAreMarkedAsFailed() {
TestData suite = new TestData(identifier);

TestIdentifier childId = TestIdentifier.from(new StubbedTestDescriptor(createId("child")));
TestPlan testPlan = Mockito.mock(TestPlan.class);
when(testPlan.getRoots()).thenReturn(Set.of(childId));

TestData childResult = new TestData(childId).started();

Document xml = generateTestXml(testPlan, suite, List.of(childResult), true);

// Because of the way we generated the XML, the root element is the `testsuite` one
Element root = xml.getDocumentElement();
assertEquals("testsuite", root.getTagName());
assertEquals("1", root.getAttribute("tests"));
assertEquals("1", root.getAttribute("failures"));

NodeList allTestCases = root.getElementsByTagName("testcase");
assertEquals(1, allTestCases.getLength());
Node testCase = allTestCases.item(0);

Node failure = getChild("failure", testCase);

assertNotNull(failure);

assertEquals("Test timed out and was interrupted", failure.getTextContent());
}

@Test
void throwablesWithNullMessageAreSerialized() {
var test = new TestData(identifier).mark(TestExecutionResult.failed(new Throwable()));
Expand Down Expand Up @@ -219,12 +250,13 @@ public void ensureOutputsAreProperlyEscaped() {
}

private Document generateTestXml(TestPlan testPlan, TestData testCase) {
return generateDocument(xml -> new TestCaseXmlRenderer(testPlan).toXml(xml, testCase));
return generateDocument(xml -> new TestCaseXmlRenderer(testPlan).toXml(xml, testCase, false));
}

private Document generateTestXml(
TestPlan testPlan, TestData suite, Collection<TestData> testCases) {
return generateDocument(xml -> new TestSuiteXmlRenderer(testPlan).toXml(xml, suite, testCases));
TestPlan testPlan, TestData suite, Collection<TestData> testCases, boolean interrupted) {
return generateDocument(
xml -> new TestSuiteXmlRenderer(testPlan).toXml(xml, suite, testCases, interrupted));
}

private Document generateDocument(XmlGenerator renderer) {
Expand Down

0 comments on commit cb7206c

Please sign in to comment.