Clean up code coverage reporting.

CTest is now run directly and `make coverage` (like `make test`) now expects you to run `make` beforehand, which is more flexible for the user. This patch also reduces clutter by properly excluding unwanted files and reduces the number of explicit exlusion regexes that are required. Gcov reports are still confusing and report very low branch coverage (which is picked up by Codecov, unfortunately), but the llvm-cov reports are nice and clean now.
keepassxreboot · Sep 28, 2021 · 7ff8720 · 7ff8720
1 parent e6bf846
commit 7ff8720
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 30 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -385,35 +385,27 @@ endif(WITH_TESTS)
 if(WITH_COVERAGE)
     # Include code coverage, use with -DCMAKE_BUILD_TYPE=Debug
     include(CodeCoverage)
-    set(COVERAGE_EXCLUDES
-            "\\(.+/\\)?tests/.\\*"
-            "\\(.+/\\)?build/.\\*"
-            "\\(.+/\\)?thirdparty/.\\*"
-            "\\(.+/\\)?CMakeFiles/.\\*"
-            "src/main.cpp"
-            ".\\*/moc_\\[^/\\]+\\.cpp"
-            ".\\*/ui_\\[^/\\]+\\.h"
-            ".\\*/\\[^/\\]+_autogen/.\\*"
-            "\\(.+/\\)?zxcvbn/.\\*"
-            "/Applications/.\\*"
-            "/opt/.\\*")
     append_coverage_compiler_flags()
 
+    set(COVERAGE_EXCLUDES
+            "'^(.+/)?(thirdparty|zxcvbn)/.*'"
+            "'^(.+/)?main\\.cpp$$'"
+            "'^(.+/)?cli/keepassxc-cli\\.cpp$$'"
+            "'^(.+/)?proxy/keepassxc-proxy\\.cpp$$'")
     if(WITH_COVERAGE AND CMAKE_COMPILER_IS_CLANGXX)
         set(MAIN_BINARIES
                 "$<TARGET_FILE:${PROGNAME}>"
                 "$<TARGET_FILE:keepassxc-cli>"
                 "$<TARGET_FILE:keepassxc-proxy>")
         setup_target_for_coverage_llvm(
                 NAME coverage
-                EXECUTABLE $(MAKE) test
                 BINARY ${MAIN_BINARIES}
-                SOURCES ${CMAKE_SOURCE_DIR}/src
+                SOURCES_ROOT ${CMAKE_SOURCE_DIR}/src
         )
     else()
         setup_target_for_coverage_gcovr(
                 NAME coverage
-                EXECUTABLE $(MAKE) test
+                SOURCES_ROOT ${CMAKE_SOURCE_DIR}/src
         )
     endif()
 endif()

diff --git a/cmake/CodeCoverage.cmake b/cmake/CodeCoverage.cmake
@@ -45,7 +45,7 @@ elseif(CMAKE_COMPILER_IS_CLANGXX)
 endif()
 
 set(CMAKE_COVERAGE_FORMAT
-    "html" "txt"
+    "html" "xml"
     CACHE STRING "Coverage report output format.")
 set_property(CACHE CMAKE_COVERAGE_FORMAT PROPERTY STRINGS "html" "txt")
 
@@ -93,7 +93,7 @@ endif()
 function(SETUP_TARGET_FOR_COVERAGE_GCOVR)
 
     set(options NONE)
-    set(oneValueArgs NAME)
+    set(oneValueArgs NAME SOURCES_ROOT)
     set(multiValueArgs EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES)
     cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
@@ -110,33 +110,48 @@ function(SETUP_TARGET_FOR_COVERAGE_GCOVR)
 
     add_custom_target(${Coverage_NAME}
         # Run tests
-        COMMAND $(MAKE)
-        COMMAND ${Coverage_EXECUTABLE}
+        COMMAND ctest -C $<CONFIG> $ENV{ARGS} $$ARGS
 
-        # Create folder
-        COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/${Coverage_NAME}
         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
         DEPENDS ${Coverage_DEPENDENCIES}
     )
 
     if("html" IN_LIST CMAKE_COVERAGE_FORMAT)
         add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
+            # Create folder
+            COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/${Coverage_NAME}-html
+
             # Running gcovr HTML
             COMMAND ${GCOVR_PATH} --html --html-details
-                -r ${PROJECT_SOURCE_DIR} ${GCOVR_EXCLUDES}
+                -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
                 --object-directory=${PROJECT_BINARY_DIR}
+                --exclude-unreachable-branches --exclude-throw-branches
                 -o ${Coverage_NAME}-html/index.html
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             COMMENT "Running gcovr to produce HTML code coverage report ${Coverage_NAME}-html."
         )
     endif()
 
+    if("xml" IN_LIST CMAKE_COVERAGE_FORMAT)
+        add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
+            # Running gcovr TXT
+            COMMAND ${GCOVR_PATH} --xml
+                -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
+                --object-directory=${PROJECT_BINARY_DIR}
+                --exclude-unreachable-branches --exclude-throw-branches
+                -o ${Coverage_NAME}.xml
+            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+            COMMENT "Running gcovr to produce XML code coverage report ${Coverage_NAME}.xml."
+        )
+    endif()
+
     if("txt" IN_LIST CMAKE_COVERAGE_FORMAT)
         add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
-            # Running gcovr XML
+            # Running gcovr TXT
             COMMAND ${GCOVR_PATH}
-                -r ${PROJECT_SOURCE_DIR} ${GCOVR_EXCLUDES}
+                -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
                 --object-directory=${PROJECT_BINARY_DIR}
+                --exclude-unreachable-branches --exclude-throw-branches
                 -o ${Coverage_NAME}.txt
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             COMMENT "Running gcovr to produce TXT code coverage report ${Coverage_NAME}.txt."
@@ -158,8 +173,8 @@ endfunction() # SETUP_TARGET_FOR_COVERAGE_GCOVR
 function(SETUP_TARGET_FOR_COVERAGE_LLVM)
 
     set(options NONE)
-    set(oneValueArgs NAME PROF_FILE)
-    set(multiValueArgs EXECUTABLE BINARY EXECUTABLE_ARGS SOURCES DEPENDENCIES)
+    set(oneValueArgs NAME SOURCES_ROOT PROF_FILE)
+    set(multiValueArgs EXECUTABLE BINARY EXECUTABLE_ARGS DEPENDENCIES)
     cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
     if(XCRUN_PATH)
@@ -191,8 +206,7 @@ function(SETUP_TARGET_FOR_COVERAGE_LLVM)
     endif()
 
     add_custom_target(${Coverage_NAME}
-        COMMAND $(MAKE)
-        COMMAND ${CMAKE_COMMAND} -E env LLVM_PROFILE_FILE=${LLVM_PROFILE_DIR}/profile-%p.profraw ${Coverage_EXECUTABLE}
+        COMMAND ${CMAKE_COMMAND} -E env LLVM_PROFILE_FILE=${LLVM_PROFILE_DIR}/profile-%p.profraw ctest -C $<CONFIG> $$ARGS
 
         COMMAND ${LLVM_PROFDATA_PATH} merge -sparse ${LLVM_PROFILE_DIR}/* -o coverage.profdata
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
@@ -201,15 +215,19 @@ function(SETUP_TARGET_FOR_COVERAGE_LLVM)
     if("html" IN_LIST CMAKE_COVERAGE_FORMAT)
         add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
             COMMAND ${LLVM_COV_PATH} show -instr-profile=coverage.profdata ${COV_BINARY}
-                --format=html --output-dir=${Coverage_NAME}-html ${COV_EXCLUDES} ${Coverage_SOURCES}
+                --format=html --output-dir=${Coverage_NAME}-html ${COV_EXCLUDES} ${Coverage_SOURCES_ROOT}
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             COMMENT "Running llvm-cov to produce HTML code coverage report ${Coverage_NAME}-html")
     endif()
 
+    if("xml" IN_LIST CMAKE_COVERAGE_FORMAT)
+        message(WARNING "XML coverage report format not supported for llvm-cov")
+    endif()
+
     if("txt" IN_LIST CMAKE_COVERAGE_FORMAT)
         add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
             COMMAND ${LLVM_COV_PATH} show -instr-profile=coverage.profdata ${COV_BINARY}
-                --format=text ${COV_EXCLUDES} ${Coverage_SOURCES} > ${Coverage_NAME}.txt
+                --format=text ${COV_EXCLUDES} ${Coverage_SOURCES_ROOT} > ${Coverage_NAME}.txt
 
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             COMMENT "Running llvm-cov to produce TXT code coverage report ${Coverage_NAME}.txt.")

diff --git a/codecov.yaml b/codecov.yaml
@@ -2,5 +2,7 @@ coverage:
   range: 60..80
   round: nearest
   precision: 2
+fixes:
+  - "*/src/::"
 comment:
   require_changes: true