Adds ReferenceTest containing duplicate test case name. Adds improved…

… error handling for discoverer so a duplicate testcase name error can be logged.
JohnnyHendriks · Jul 6, 2018 · 9fed124 · 9fed124
1 parent dd04bc6
commit 9fed124
Show file tree

Hide file tree

Showing 13 changed files with 190 additions and 14 deletions.
diff --git a/Docs/CHANGELOG.md b/Docs/CHANGELOG.md
@@ -10,4 +10,4 @@
 ### Extended Features
 
  - Added `Debug` level option to `<Logging>` setting and updated what is logged in the `Verbose` level.
-
+ - Improve error handling, specifically when a duplicate testname is used, the potential resulting error is logged. 
diff --git a/Docs/Settings.md b/Docs/Settings.md
@@ -131,8 +131,16 @@ int main(int argc, char* argv[])
     // Check if custom discovery needs to be performed
     if(doDiscover)
     {
-        Discover(session);
-        return 0;
+        try
+        {
+            Discover(session);
+            return 0;
+        }
+        catch( std::exception& ex )
+        {
+            Catch::cerr() << ex.what() << std::endl;
+            return Catch::MaxExitCode;
+        }
     }
 
     // Let Catch2 do its thing

diff --git a/Libraries/Catch2Interface/Discoverer.cs b/Libraries/Catch2Interface/Discoverer.cs
@@ -142,10 +142,12 @@ private string GetTestCaseInfo(string source)
             process.StartInfo.Arguments = _settings.DiscoverCommandLine;
             process.StartInfo.CreateNoWindow = true;
             process.StartInfo.RedirectStandardOutput = true;
+            process.StartInfo.RedirectStandardError = true;
             process.StartInfo.UseShellExecute = false;
             process.Start();
 
             var output = process.StandardOutput.ReadToEndAsync();
+            var erroroutput = process.StandardError.ReadToEndAsync();
 
             if(_settings.DiscoverTimeout > 0)
             {
@@ -159,13 +161,27 @@ private string GetTestCaseInfo(string source)
             if( !process.HasExited )
             {
                 process.Kill();
-                _logbuilder.Append($"  Killed process. Threw away following output:{Environment.NewLine}{output.Result}");
+                LogVerbose($"  Killed process. Threw away following output:{Environment.NewLine}{output.Result}");
                 return string.Empty;
             }
             else
             {
-                // Extract tests
-                return output.Result;
+                if(!string.IsNullOrEmpty(erroroutput.Result))
+                {
+                    LogNormal($"  Error Occured (exit code {process.ExitCode}):{Environment.NewLine}{erroroutput.Result}");
+                    LogDebug($"  output:{Environment.NewLine}{output.Result}");
+                    return string.Empty;
+                }
+
+                if (!string.IsNullOrEmpty(output.Result))
+                {
+                    return output.Result;
+                }
+                else
+                {
+                    LogDebug($"  No output{Environment.NewLine}");
+                    return string.Empty;
+                }
             }
         }
 

diff --git a/Libraries/Catch2TestAdapter/TestDiscoverer.cs b/Libraries/Catch2TestAdapter/TestDiscoverer.cs
@@ -63,7 +63,7 @@ public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discove
                 return;
             }
 
-            if (_settings.FilenameFilter == string.Empty)
+            if (string.IsNullOrEmpty(_settings.FilenameFilter))
             {
                 LogNormal(TestMessageLevel.Error, Resources.ErrorStrings.SettingsEmptyFilenameFilter);
                 return;
@@ -86,7 +86,7 @@ private void DiscoverTests(IEnumerable<string> sources)
             var discoverer = new Catch2Interface.Discoverer(_settings);
 
             var testcases = discoverer.GetTests(sources);
-            if(discoverer.Log != null && discoverer.Log != string.Empty)
+            if(!string.IsNullOrEmpty(discoverer.Log))
             {
                 LogNormal(TestMessageLevel.Informational, $"Discover log:{Environment.NewLine}{discoverer.Log}");
             }

diff --git a/Libraries/Catch2TestAdapter/TestExecutor.cs b/Libraries/Catch2TestAdapter/TestExecutor.cs
@@ -133,10 +133,9 @@ private List<TestCase> GetTests(IEnumerable<string> sources)
 
             var discoverer = new Catch2Interface.Discoverer(_settings);
 
-
             var testcases = discoverer.GetTests(sources);
 
-            if (discoverer.Log != null && discoverer.Log != string.Empty)
+            if (!string.IsNullOrEmpty(discoverer.Log))
             {
                 LogNormal(TestMessageLevel.Informational, $"Discover log:{Environment.NewLine}{discoverer.Log}");
             }
@@ -196,7 +195,7 @@ private TestResult RunTest(TestCase test)
             {
                 var testresult = _executor.Run(test.DisplayName, test.Source);
 
-                if(_executor.Log != null && _executor.Log != string.Empty)
+                if(!string.IsNullOrEmpty(_executor.Log))
                 {
                     LogNormal(TestMessageLevel.Informational, $"Executor log:{Environment.NewLine}{_executor.Log}");
                 }

diff --git a/ReferenceTests/MSBuild/Projects/Catch_Testset04/Catch_Testset04.vcxproj b/ReferenceTests/MSBuild/Projects/Catch_Testset04/Catch_Testset04.vcxproj
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\Catch2Unittest.targets" />
+  <PropertyGroup Label="Globals">
+    <RootNamespace>Catch_Testset04</RootNamespace>
+    <ProjectTypeGuids>{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}</ProjectTypeGuids>
+    <ProjectGuid>{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}</ProjectGuid>
+  </PropertyGroup>
+  <PropertyGroup>
+    <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
+  </PropertyGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\Src\Catch2\Catch_Testset04\UT_Tests01.cpp" />
+    <ClCompile Include="..\..\..\Src\Catch2\main.cpp" />
+  </ItemGroup>
+</Project>
diff --git a/ReferenceTests/MSBuild/Projects/Catch_Testset04/Catch_Testset04.vcxproj.filters b/ReferenceTests/MSBuild/Projects/Catch_Testset04/Catch_Testset04.vcxproj.filters
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Files">
+      <UniqueIdentifier>{e9f92384-33de-4567-ac81-601cc1c65521}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="Files\Catch">
+      <UniqueIdentifier>{36abadce-429e-4b62-9e22-98a0b1ba6ed1}</UniqueIdentifier>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\Src\Catch2\main.cpp">
+      <Filter>Files\Catch</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\Src\Catch2\Catch_Testset04\UT_Tests01.cpp">
+      <Filter>Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+</Project>
diff --git a/ReferenceTests/ReferenceTests.sln b/ReferenceTests/ReferenceTests.sln
@@ -11,6 +11,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Catch_Testset03", "MSBuild\
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Catch_Dummy", "MSBuild\Projects\Catch_Dummy\Catch_Dummy.vcxproj", "{EA4A2711-386D-4A1B-9234-CBFFE3D23307}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Catch_Testset04", "MSBuild\Projects\Catch_Testset04\Catch_Testset04.vcxproj", "{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -51,6 +53,14 @@ Global
 		{EA4A2711-386D-4A1B-9234-CBFFE3D23307}.Release|x64.Build.0 = Release|x64
 		{EA4A2711-386D-4A1B-9234-CBFFE3D23307}.Release|x86.ActiveCfg = Release|Win32
 		{EA4A2711-386D-4A1B-9234-CBFFE3D23307}.Release|x86.Build.0 = Release|Win32
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Debug|x64.ActiveCfg = Debug|x64
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Debug|x64.Build.0 = Debug|x64
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Debug|x86.ActiveCfg = Debug|Win32
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Debug|x86.Build.0 = Debug|Win32
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|x64.ActiveCfg = Release|x64
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|x64.Build.0 = Release|x64
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|x86.ActiveCfg = Release|Win32
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|x86.Build.0 = Release|Win32
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

diff --git a/ReferenceTests/Src/Catch2/Catch_Testset04/UT_Tests01.cpp b/ReferenceTests/Src/Catch2/Catch_Testset04/UT_Tests01.cpp
@@ -0,0 +1,60 @@
+/** Basic Info **
+
+Copyright: 2018 Johnny Hendriks
+
+Author : Johnny Hendriks
+Year   : 2018
+Project: VSTestAdapter for Catch2
+Licence: MIT
+
+Notes: None
+
+** Basic Info **/
+
+/************
+ * Includes *
+ ************/
+
+// Catch2
+#include <catch.hpp>
+
+
+/**************
+ * Start code *
+ **************/
+
+namespace CatchTestset01
+{
+    TEST_CASE( "Testset04.Tests01. Duplicate", "[Passing]" )
+    {
+        int x = 42;
+        int y = 42;
+
+        INFO("basic");
+
+        CHECK( x == y );
+        CHECK( y == x );
+
+        REQUIRE( x <= y );
+        REQUIRE( y >= x );
+    }
+
+    TEST_CASE( "Testset04.Tests01. Duplicate", "[Failing]" )
+    {
+        int x = 42;
+        int y = 47;
+
+        INFO("basic");
+
+        CHECK( x == y );
+        CHECK( y == x );
+
+        REQUIRE( x <= y );
+        REQUIRE( y >= x );
+    }
+
+} // End namespace: CatchTestset01
+
+/************
+ * End code *
+ ************/
diff --git a/ReferenceTests/Src/Catch2/main.cpp b/ReferenceTests/Src/Catch2/main.cpp
@@ -44,8 +44,16 @@ int main(int argc, char* argv[])
 
     if(doDiscover)
     {
-        Discover(session);
-        return 0;
+        try
+        {
+            Discover(session);
+            return 0;
+        }
+        catch( std::exception& ex )
+        {
+            Catch::cerr() << ex.what() << std::endl;
+            return Catch::MaxExitCode;
+        }
     }
 
     return session.run();

diff --git a/UnitTests/UT_Catch2Interface/DiscovererTests.cs b/UnitTests/UT_Catch2Interface/DiscovererTests.cs
@@ -131,6 +131,23 @@ public void TestGetTestsDefaultTag1NoHidden()
             Assert.AreEqual(2, tests.Count);
         }
 
+        [TestMethod]
+        public void TestGetTestsDuplicateTestname()
+        {
+            var settings = new Settings();
+            settings.DiscoverCommandLine = "--list-tests";
+            settings.FilenameFilter = ".*";
+            settings.IncludeHidden = false;
+
+            var discoverer = new Discoverer(settings);
+            string[] sources = { Path_Testset04 };
+            var tests = discoverer.GetTests(sources) as List<TestCase>;
+
+            Assert.AreEqual(0, tests.Count);
+            Assert.IsFalse(string.IsNullOrEmpty(discoverer.Log));
+            Assert.IsTrue(discoverer.Log.Contains("Error Occured"));
+        }
+
         [TestMethod]
         public void TestGetTestsXml()
         {
@@ -224,6 +241,16 @@ private string Path_Testset02
             }
         }
 
+        // Contains duplicate name
+        private string Path_Testset04
+        {
+            get
+            {
+                string path = TestContext.TestRunDirectory + @"\..\..\ReferenceTests\_unittest64\Release\Catch_Testset04.exe";
+                return Path.GetFullPath(path);
+            }
+        }
+
         #endregion // Helper Methods
     }
 }
diff --git a/VSIX/VSTestAdapterCatch2/source.extension.vsixmanifest b/VSIX/VSTestAdapterCatch2/source.extension.vsixmanifest
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <PackageManifest Version="2.0.0" xmlns="http://schemas.microsoft.com/developer/vsx-schema/2011" xmlns:d="http://schemas.microsoft.com/developer/vsx-schema-design/2011">
     <Metadata>
-        <Identity Id="VSTestAdapterCatch2.73ba8471-3771-47bf-bd23-49a1ba09af89" Version="1.0.0" Language="en-US" Publisher="Johnny Hendriks" />
+        <Identity Id="VSTestAdapterCatch2.73ba8471-3771-47bf-bd23-49a1ba09af89" Version="1.0.1" Language="en-US" Publisher="Johnny Hendriks" />
         <DisplayName>Test Adapter for Catch2</DisplayName>
         <Description xml:space="preserve">Test Adapter for use with the Catch2 C++ unit test framework.</Description>
         <MoreInfo>https://github.com/JohnnyHendriks/TestAdapter_Catch2</MoreInfo>

diff --git a/VSTestAdapterCatch2.sln b/VSTestAdapterCatch2.sln
@@ -27,6 +27,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Catch2TestAdapter", "Librar
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VSTestAdapterCatch2", "VSIX\VSTestAdapterCatch2\VSTestAdapterCatch2.csproj", "{289EE75E-CBB6-4179-B61C-8EED1554CFF7}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Catch_Testset04", "ReferenceTests\MSBuild\Projects\Catch_Testset04\Catch_Testset04.vcxproj", "{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -125,6 +127,16 @@ Global
 		{289EE75E-CBB6-4179-B61C-8EED1554CFF7}.Release|x64.Build.0 = Release|Any CPU
 		{289EE75E-CBB6-4179-B61C-8EED1554CFF7}.Release|x86.ActiveCfg = Release|Any CPU
 		{289EE75E-CBB6-4179-B61C-8EED1554CFF7}.Release|x86.Build.0 = Release|Any CPU
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Debug|x64.ActiveCfg = Debug|x64
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Debug|x64.Build.0 = Debug|x64
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Debug|x86.ActiveCfg = Debug|Win32
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Debug|x86.Build.0 = Debug|Win32
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|Any CPU.ActiveCfg = Release|Win32
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|x64.ActiveCfg = Release|x64
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|x64.Build.0 = Release|x64
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|x86.ActiveCfg = Release|Win32
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A}.Release|x86.Build.0 = Release|Win32
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -138,6 +150,7 @@ Global
 		{080FBAE5-1808-4022-A1AF-14F43D8E6030} = {B4A54678-F45D-4252-8CDF-9D549E42797A}
 		{30A41C70-7668-40F7-9DC3-1870F440ABB1} = {B4A54678-F45D-4252-8CDF-9D549E42797A}
 		{289EE75E-CBB6-4179-B61C-8EED1554CFF7} = {D2773C29-88F8-4B67-9874-4534A47A9D74}
+		{6F30DB9F-419F-4EAC-AAFC-17C97C94DA9A} = {C04B9095-2EFB-4974-88A7-D3DC051C98BF}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {EA93387E-2359-4A31-9AE3-693830AEC329}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -10,4 +10,4 @@
		### Extended Features

		- Added `Debug` level option to `<Logging>` setting and updated what is logged in the `Verbose` level.

		- Improve error handling, specifically when a duplicate testname is used, the potential resulting error is logged.