Skip to content

Commit

Permalink
gtest: Output a canned test case for test suite setup / teardown fail…
Browse files Browse the repository at this point in the history 
…ures in XML/JSON

This surfaces useful information about the environment failure in a structured form.

As we can see from the updated test, previously unsurfaced information is now present.

PiperOrigin-RevId: 709892315
Change-Id: I2656294d50c33f995bef5c96195a66cff3c4b907
  • Loading branch information
@copybara-github
Abseil Team authored and copybara-github committed Dec 26, 2024
1 parent e54519b commit 7d76a23
Show file tree
Hide file tree
Showing 4 changed files with 198 additions and 28 deletions.
50 changes: 44 additions & 6 deletions googletest/src/gtest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3989,6 +3989,12 @@ class XmlUnitTestResultPrinter : public EmptyTestEventListener {
static void OutputXmlTestSuiteForTestResult(::std::ostream* stream,
const TestResult& result);

// Streams a test case XML stanza containing the given test result.
//
// Requires: result.Failed()
static void OutputXmlTestCaseForTestResult(::std::ostream* stream,
const TestResult& result);

// Streams an XML representation of a TestResult object.
static void OutputXmlTestResult(::std::ostream* stream,
const TestResult& result);
Expand Down Expand Up @@ -4236,6 +4242,15 @@ void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult(
FormatEpochTimeInMillisAsIso8601(result.start_timestamp()));
*stream << ">";

OutputXmlTestCaseForTestResult(stream, result);

// Complete the test suite.
*stream << " </testsuite>\n";
}

// Streams a test case XML stanza containing the given test result.
void XmlUnitTestResultPrinter::OutputXmlTestCaseForTestResult(
::std::ostream* stream, const TestResult& result) {
// Output the boilerplate for a minimal test case with a single test.
*stream << " <testcase";
OutputXmlAttribute(stream, "testcase", "name", "");
Expand All @@ -4250,9 +4265,6 @@ void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult(

// Output the actual test result.
OutputXmlTestResult(stream, result);

// Complete the test suite.
*stream << " </testsuite>\n";
}

// Prints an XML representation of a TestInfo object.
Expand Down Expand Up @@ -4379,6 +4391,10 @@ void XmlUnitTestResultPrinter::PrintXmlTestSuite(std::ostream* stream,
if (test_suite.GetTestInfo(i)->is_reportable())
OutputXmlTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i));
}
if (test_suite.ad_hoc_test_result().Failed()) {
OutputXmlTestCaseForTestResult(stream, test_suite.ad_hoc_test_result());
}

*stream << " </" << kTestsuite << ">\n";
}

Expand Down Expand Up @@ -4518,6 +4534,12 @@ class JsonUnitTestResultPrinter : public EmptyTestEventListener {
static void OutputJsonTestSuiteForTestResult(::std::ostream* stream,
const TestResult& result);

// Streams a test case JSON stanza containing the given test result.
//
// Requires: result.Failed()
static void OutputJsonTestCaseForTestResult(::std::ostream* stream,
const TestResult& result);

// Streams a JSON representation of a TestResult object.
static void OutputJsonTestResult(::std::ostream* stream,
const TestResult& result);
Expand Down Expand Up @@ -4688,6 +4710,15 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult(
}
*stream << Indent(6) << "\"testsuite\": [\n";

OutputJsonTestCaseForTestResult(stream, result);

// Finish the test suite.
*stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}";
}

// Streams a test case JSON stanza containing the given test result.
void JsonUnitTestResultPrinter::OutputJsonTestCaseForTestResult(
::std::ostream* stream, const TestResult& result) {
// Output the boilerplate for a new test case.
*stream << Indent(8) << "{\n";
OutputJsonKey(stream, "testcase", "name", "", Indent(10));
Expand All @@ -4704,9 +4735,6 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult(

// Output the actual test result.
OutputJsonTestResult(stream, result);

// Finish the test suite.
*stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}";
}

// Prints a JSON representation of a TestInfo object.
Expand Down Expand Up @@ -4851,6 +4879,16 @@ void JsonUnitTestResultPrinter::PrintJsonTestSuite(
OutputJsonTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i));
}
}

// If there was a failure in the test suite setup or teardown include that in
// the output.
if (test_suite.ad_hoc_test_result().Failed()) {
if (comma) {
*stream << ",\n";
}
OutputJsonTestCaseForTestResult(stream, test_suite.ad_hoc_test_result());
}

*stream << "\n" << kIndent << "]\n" << Indent(4) << "}";
}

Expand Down
95 changes: 86 additions & 9 deletions googletest/test/googletest-json-output-unittest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
STACK_TRACE_TEMPLATE = '\n'

EXPECTED_NON_EMPTY = {
'tests': 26,
'tests': 28,
'failures': 5,
'disabled': 2,
'errors': 0,
Expand Down Expand Up @@ -419,6 +419,83 @@
},
],
},
{
'name': 'SetupFailTest',
'tests': 1,
'failures': 0,
'disabled': 0,
'errors': 0,
'time': '*',
'timestamp': '*',
'testsuite': [
{
'name': 'NoopPassingTest',
'file': 'gtest_xml_output_unittest_.cc',
'line': 168,
'status': 'RUN',
'result': 'SKIPPED',
'timestamp': '*',
'time': '*',
'classname': 'SetupFailTest',
'skipped': [
{'message': 'gtest_xml_output_unittest_.cc:*\n'}
],
},
{
'name': '',
'status': 'RUN',
'result': 'COMPLETED',
'timestamp': '*',
'time': '*',
'classname': '',
'failures': [{
'failure': (
'gtest_xml_output_unittest_.cc:*\nExpected equality'
' of these values:\n 1\n 2'
+ STACK_TRACE_TEMPLATE
),
'type': '',
}],
},
],
},
{
'name': 'TearDownFailTest',
'tests': 1,
'failures': 0,
'disabled': 0,
'errors': 0,
'timestamp': '*',
'time': '*',
'testsuite': [
{
'name': 'NoopPassingTest',
'file': 'gtest_xml_output_unittest_.cc',
'line': 175,
'status': 'RUN',
'result': 'COMPLETED',
'timestamp': '*',
'time': '*',
'classname': 'TearDownFailTest',
},
{
'name': '',
'status': 'RUN',
'result': 'COMPLETED',
'timestamp': '*',
'time': '*',
'classname': '',
'failures': [{
'failure': (
'gtest_xml_output_unittest_.cc:*\nExpected equality'
' of these values:\n 1\n 2'
+ STACK_TRACE_TEMPLATE
),
'type': '',
}],
},
],
},
{
'name': 'TypedTest/0',
'tests': 1,
Expand All @@ -431,7 +508,7 @@
'name': 'HasTypeParamAttribute',
'type_param': 'int',
'file': 'gtest_xml_output_unittest_.cc',
'line': 173,
'line': 189,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
Expand All @@ -451,7 +528,7 @@
'name': 'HasTypeParamAttribute',
'type_param': 'long',
'file': 'gtest_xml_output_unittest_.cc',
'line': 173,
'line': 189,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
Expand All @@ -471,7 +548,7 @@
'name': 'HasTypeParamAttribute',
'type_param': 'int',
'file': 'gtest_xml_output_unittest_.cc',
'line': 180,
'line': 196,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
Expand All @@ -491,7 +568,7 @@
'name': 'HasTypeParamAttribute',
'type_param': 'long',
'file': 'gtest_xml_output_unittest_.cc',
'line': 180,
'line': 196,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
Expand All @@ -512,7 +589,7 @@
'name': 'HasValueParamAttribute/0',
'value_param': '33',
'file': 'gtest_xml_output_unittest_.cc',
'line': 164,
'line': 180,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
Expand All @@ -523,7 +600,7 @@
'name': 'HasValueParamAttribute/1',
'value_param': '42',
'file': 'gtest_xml_output_unittest_.cc',
'line': 164,
'line': 180,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
Expand All @@ -534,7 +611,7 @@
'name': 'AnotherTestThatHasValueParamAttribute/0',
'value_param': '33',
'file': 'gtest_xml_output_unittest_.cc',
'line': 165,
'line': 181,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
Expand All @@ -545,7 +622,7 @@
'name': 'AnotherTestThatHasValueParamAttribute/1',
'value_param': '42',
'file': 'gtest_xml_output_unittest_.cc',
'line': 165,
'line': 181,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
Expand Down
65 changes: 52 additions & 13 deletions googletest/test/gtest_xml_output_unittest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Unit test for the gtest_xml_output module"""
"""Unit test for the gtest_xml_output module."""

import datetime
import errno
import os
import re
import sys
from xml.dom import minidom, Node
from xml.dom import minidom

from googletest.test import gtest_test_utils
from googletest.test import gtest_xml_test_utils
Expand Down Expand Up @@ -67,7 +67,7 @@
sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)

EXPECTED_NON_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="26" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuites tests="28" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="Succeeds" file="gtest_xml_output_unittest_.cc" line="53" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
</testsuite>
Expand Down Expand Up @@ -173,23 +173,44 @@
</properties>
</testcase>
</testsuite>
<testsuite name="SetupFailTest" tests="1" failures="0" disabled="0" skipped="1" errors="0" time="*" timestamp="*">
<testcase name="NoopPassingTest" file="gtest_xml_output_unittest_.cc" line="168" status="run" result="skipped" time="*" timestamp="*" classname="SetupFailTest">
<skipped message="gtest_xml_output_unittest_.cc:*&#x0A;"><![CDATA[gtest_xml_output_unittest_.cc:*
]]></skipped>
</testcase>
<testcase name="" status="run" result="completed" classname="" time="*" timestamp="*">
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2%(stack_entity)s" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
Expected equality of these values:
1
2%(stack)s]]></failure>
</testcase>
</testsuite>
<testsuite name="TearDownFailTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="NoopPassingTest" file="gtest_xml_output_unittest_.cc" line="175" status="run" result="completed" time="*" timestamp="*" classname="TearDownFailTest"/>
<testcase name="" status="run" result="completed" classname="" time="*" timestamp="*">
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2%(stack_entity)s" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
Expected equality of these values:
1
2%(stack)s]]></failure>
</testcase>
</testsuite>
<testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="164" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="HasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="164" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="165" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="165" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="180" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="HasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="180" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="181" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="181" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
</testsuite>
<testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="173" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="189" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" />
</testsuite>
<testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="173" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="189" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" />
</testsuite>
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="180" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="196" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
</testsuite>
<testsuite name="Single/TypeParameterizedTestSuite/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="180" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="196" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" />
</testsuite>
</testsuites>""" % {
'stack': STACK_TRACE_TEMPLATE,
Expand All @@ -205,6 +226,24 @@
</testsuite>
</testsuites>"""

ACTUAL_OUTPUT = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" ad_hoc_property="42" name="AllTests">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="Succeeds" file="gtest_xml_output_unittest_.cc" line="53" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
</testsuite>
<testsuite name="PropertyRecordingTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*" SetUpTestSuite="yes" TearDownTestSuite="aye">
<testcase name="IntValuedProperty" file="gtest_xml_output_unittest_.cc" line="125" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest">
<properties>
<property name="key_int" value="1"/>
</properties>
</testcase>
</testsuite>
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" type_param="int" file="gtest_xml_output_unittest_.cc" line="196" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0"/>
</testsuite>
</testsuites>
"""

EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
Expand All @@ -217,8 +256,8 @@
</properties>
</testcase>
</testsuite>
<testsuite name="Single/ValueParamTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="164" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" type_param="*" file="gtest_xml_output_unittest_.cc" line="196" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
</testsuite>
</testsuites>"""

Expand Down
Loading

0 comments on commit 7d76a23

Please sign in to comment.