Skip to content

Commit

Permalink
Add support for reportng Fatal Error Conditions. Fix bug Warnings in …
Browse files Browse the repository at this point in the history
…sections not always displayed. Improve robustness of runner output parsing.
  • Loading branch information
JohnnyHendriks committed Jul 12, 2018
1 parent 9fed124 commit 7bdf548
Show file tree
Hide file tree
Showing 20 changed files with 725 additions and 26 deletions.
8 changes: 7 additions & 1 deletion Docs/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,10 @@
### Extended Features

- Added `Debug` level option to `<Logging>` setting and updated what is logged in the `Verbose` level.
- Improve error handling, specifically when a duplicate testname is used, the potential resulting error is logged.
- Improve error handling, specifically when a duplicate testname is used, the potential resulting error is logged.
- Add support for reporting Fatal Error Conditions

### Bug fixes

- Bug: Warnings in Sections are not displayed when there are no failures. Fixed.
- Bug: Get invalid test runner output error when Catch2 xml output produces additional text after xml report. Additional text is now ignored.
1 change: 1 addition & 0 deletions Libraries/Catch2Interface/Catch2Interface.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
<Compile Include="Constants.cs" />
<Compile Include="Discoverer.cs" />
<Compile Include="Executor.cs" />
<Compile Include="Reporter\FatalErrorCondition.cs" />
<Compile Include="Reporter\Exception.cs" />
<Compile Include="Reporter\Expression.cs" />
<Compile Include="Reporter\Warning.cs" />
Expand Down
1 change: 1 addition & 0 deletions Libraries/Catch2Interface/Constants.cs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public static class Constants
public const string NodeName_Exception = "Exception";
public const string NodeName_Expression = "Expression";
public const string NodeName_Failure = "Failure";
public const string NodeName_FatalErrorCondition = "FatalErrorCondition";
public const string NodeName_Info = "Info";
public const string NodeName_OverallResult = "OverallResult";
public const string NodeName_OverallResults = "OverallResults";
Expand Down
76 changes: 76 additions & 0 deletions Libraries/Catch2Interface/Reporter/FatalErrorCondition.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/** Basic Info **
Copyright: 2018 Johnny Hendriks
Author : Johnny Hendriks
Year : 2018
Project: VSTestAdapter for Catch2
Licence: MIT
Notes: None
** Basic Info **/

using System;
using System.IO;
using System.Text;
using System.Xml;

namespace Catch2Interface.Reporter
{

/*YAML
Class :
Description : >
This class is intended to handle the xml "FatalErrorCondition" node in a Catch2 Xml report.
*/
public class FatalErrorCondition
{
#region Properties

public string Filename { get; private set; }
public int Line { get; private set; } = -1;
public string Message { get; private set; }

#endregion // Properties

#region Constructor

public FatalErrorCondition(XmlNode node)
{
if (node.Name == Constants.NodeName_FatalErrorCondition)
{
int line;
if (int.TryParse(node.Attributes["line"].Value, out line))
{
Line = line;
}

Filename = node.Attributes["filename"]?.Value;

var nodeMessage = node?.FirstChild;
if (nodeMessage != null && nodeMessage.NodeType == XmlNodeType.Text)
{
Message = nodeMessage.Value.Trim();
}
}
}

#endregion // Constructor

#region Public Methods

public string GenerateFailureInfo()
{
StringBuilder msg = new StringBuilder();

msg.Append($"{Path.GetFileName(Filename)} line {Line}: Failed{Environment.NewLine}");
msg.Append($"due to a fatal error condition with message:{Environment.NewLine}");
msg.Append($" {Message}{Environment.NewLine}");

return msg.ToString();
}

#endregion // Public Methods
}
}
43 changes: 38 additions & 5 deletions Libraries/Catch2Interface/Reporter/Section.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,33 @@ public class Section
public string Name { get; set; }
public OverallResults OverallResults { get; set; }

public bool HasFailuresOrWarnings
{
get
{
// Check for failures
if( OverallResults == null ) return false;
if( OverallResults.Failures > 0) return true;

// Check for warnings
foreach(var child in Children)
{
switch(child)
{
case Section section:
if(section.HasFailuresOrWarnings) return true;
break;
case Warning warning:
return true;
default:
break;
}
}

return false;
}
}

#endregion // Properties

#region Constructor
Expand Down Expand Up @@ -59,17 +86,23 @@ public Section(XmlNode node)
case Constants.NodeName_Expression:
Children.Add(new Expression(child));
break;
case Constants.NodeName_Failure:
Children.Add(new Failure(child));
break;
case Constants.NodeName_FatalErrorCondition:
Children.Add(new FatalErrorCondition(child));
break;
case Constants.NodeName_Info:
Children.Add(new Info(child));
break;
case Constants.NodeName_OverallResults:
OverallResults = new OverallResults(child);
break;
case Constants.NodeName_Section:
Children.Add(new Section(child));
break;
case Constants.NodeName_Failure:
Children.Add(new Failure(child));
break;
case Constants.NodeName_OverallResults:
OverallResults = new OverallResults(child);
case Constants.NodeName_Warning:
Children.Add(new Warning(child));
break;
default:
break;
Expand Down
15 changes: 9 additions & 6 deletions Libraries/Catch2Interface/Reporter/TestCase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -64,18 +64,21 @@ public TestCase(XmlNode node)
case Constants.NodeName_Expression:
Children.Add(new Expression(child));
break;
case Constants.NodeName_Info:
Children.Add(new Info(child));
break;
case Constants.NodeName_Section:
Children.Add(new Section(child));
break;
case Constants.NodeName_Failure:
Children.Add(new Failure(child));
break;
case Constants.NodeName_FatalErrorCondition:
Children.Add(new FatalErrorCondition(child));
break;
case Constants.NodeName_Info:
Children.Add(new Info(child));
break;
case Constants.NodeName_OverallResult:
OverallResult = new OverallResult(child);
break;
case Constants.NodeName_Section:
Children.Add(new Section(child));
break;
case Constants.NodeName_Warning:
Children.Add(new Warning(child));
break;
Expand Down
63 changes: 50 additions & 13 deletions Libraries/Catch2Interface/TestResult.cs
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,27 @@ private void AppendFailure(Reporter.Failure failure)
ResetInfo();
}

private void AppendFatalErrorCondition(Reporter.FatalErrorCondition fatal)
{
AppendToStackTrace(fatal.Filename, fatal.Line);
_msgbuilder.Append(fatal.GenerateFailureInfo());
if (_infobuilder.Length > 0)
{
if (_infocount > 1)
{
_msgbuilder.Append($"with additional messages:{Environment.NewLine}");
}
else
{
_msgbuilder.Append($"with additional message:{Environment.NewLine}");
}
_msgbuilder.Append(_infobuilder.ToString());
}
_msgbuilder.AppendLine();

ResetInfo();
}

private void AppendSection(Reporter.Section section)
{
_msgbuilder.Append($"Start Section: {section.Name}{Environment.NewLine}");
Expand All @@ -170,9 +191,6 @@ private void AppendSection(Reporter.Section section)
{
switch( child )
{
case Reporter.Info info:
AppendInfo(info.Message);
break;
case Reporter.Exception exception:
AppendException(exception);
break;
Expand All @@ -182,8 +200,14 @@ private void AppendSection(Reporter.Section section)
case Reporter.Failure failure:
AppendFailure(failure);
break;
case Reporter.FatalErrorCondition fatal:
AppendFatalErrorCondition(fatal);
break;
case Reporter.Info info:
AppendInfo(info.Message);
break;
case Reporter.Section innersection:
if (innersection.OverallResults.Failures > 0)
if (innersection.HasFailuresOrWarnings)
{
AppendSection(innersection);
}
Expand Down Expand Up @@ -236,9 +260,6 @@ private void ExtractMessages()
{
switch( child )
{
case Reporter.Info info:
AppendInfo(info.Message);
break;
case Reporter.Exception exception:
AppendException(exception);
break;
Expand All @@ -248,8 +269,14 @@ private void ExtractMessages()
case Reporter.Failure failure:
AppendFailure(failure);
break;
case Reporter.FatalErrorCondition fatal:
AppendFatalErrorCondition(fatal);
break;
case Reporter.Info info:
AppendInfo(info.Message);
break;
case Reporter.Section section:
if (section.OverallResults.Failures > 0)
if (section.HasFailuresOrWarnings)
{
AppendSection(section);
}
Expand Down Expand Up @@ -332,11 +359,21 @@ private void ProcessXml()
try
{
var xml = new XmlDocument();
xml.LoadXml(_xmloutput);

var nodeGroup = xml.SelectSingleNode("Catch/Group");

ExtractTestResult(nodeGroup);
// Determine the part of the xmloutput string to parse
// In some cases Catch2 output contains additional lines of output after the
// xml-output. The XmlDocument parser doesn't like this so let's make sure those
// extra lines are ignored.
var idx = _xmloutput.IndexOf(@"</Catch>"); // Find first occurance of </Catch>
if(idx == -1) // Make sure closing tag was found
{
SetInvalidTestRunnerOutput();
}
else
{
xml.LoadXml(_xmloutput.Substring(0,idx+8));
var nodeGroup = xml.SelectSingleNode("Catch/Group");
ExtractTestResult(nodeGroup);
}
}
catch
{
Expand Down
21 changes: 21 additions & 0 deletions ReferenceTests/MSBuild/Projects/Catch_NoSEH/Catch_NoSEH.vcxproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\Catch2Unittest.targets" />
<PropertyGroup Label="Globals">
<RootNamespace>Catch_NoSEH</RootNamespace>
<ProjectTypeGuids>{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}</ProjectTypeGuids>
<ProjectGuid>{93AAED31-C6A6-4BFE-ABED-D750C575BA10}</ProjectGuid>
</PropertyGroup>
<PropertyGroup>
<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
</PropertyGroup>
<ItemDefinitionGroup>
<ClCompile>
<PreprocessorDefinitions>CATCH_CONFIG_NO_WINDOWS_SEH;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\..\Src\Catch2\Catch_NoSEH\UT_Tests01.cpp" />
<ClCompile Include="..\..\..\Src\Catch2\main.cpp" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Files">
<UniqueIdentifier>{e9f92384-33de-4567-ac81-601cc1c65521}</UniqueIdentifier>
</Filter>
<Filter Include="Files\Catch">
<UniqueIdentifier>{36abadce-429e-4b62-9e22-98a0b1ba6ed1}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\Src\Catch2\main.cpp">
<Filter>Files\Catch</Filter>
</ClCompile>
<ClCompile Include="..\..\..\Src\Catch2\Catch_NoSEH\UT_Tests01.cpp">
<Filter>Files</Filter>
</ClCompile>
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
<ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests02.cpp" />
<ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests03.cpp" />
<ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests04.cpp" />
<ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests05.cpp" />
<ClCompile Include="..\..\..\Src\Catch2\main.cpp" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,8 @@
<ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests04.cpp">
<Filter>Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests05.cpp">
<Filter>Files</Filter>
</ClCompile>
</ItemGroup>
</Project>
10 changes: 10 additions & 0 deletions ReferenceTests/ReferenceTests.sln
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Catch_Dummy", "MSBuild\Proj
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Catch_Testset04", "MSBuild\Projects\Catch_Testset04\Catch_Testset04.vcxproj", "{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Catch_NoSEH", "MSBuild\Projects\Catch_NoSEH\Catch_NoSEH.vcxproj", "{93AAED31-C6A6-4BFE-ABED-D750C575BA10}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64
Expand Down Expand Up @@ -61,6 +63,14 @@ Global
{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|x64.Build.0 = Release|x64
{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|x86.ActiveCfg = Release|Win32
{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|x86.Build.0 = Release|Win32
{93AAED31-C6A6-4BFE-ABED-D750C575BA10}.Debug|x64.ActiveCfg = Debug|x64
{93AAED31-C6A6-4BFE-ABED-D750C575BA10}.Debug|x64.Build.0 = Debug|x64
{93AAED31-C6A6-4BFE-ABED-D750C575BA10}.Debug|x86.ActiveCfg = Debug|Win32
{93AAED31-C6A6-4BFE-ABED-D750C575BA10}.Debug|x86.Build.0 = Debug|Win32
{93AAED31-C6A6-4BFE-ABED-D750C575BA10}.Release|x64.ActiveCfg = Release|x64
{93AAED31-C6A6-4BFE-ABED-D750C575BA10}.Release|x64.Build.0 = Release|x64
{93AAED31-C6A6-4BFE-ABED-D750C575BA10}.Release|x86.ActiveCfg = Release|Win32
{93AAED31-C6A6-4BFE-ABED-D750C575BA10}.Release|x86.Build.0 = Release|Win32
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
Loading

0 comments on commit 7bdf548

Please sign in to comment.