From a9b7371f16eed2bacaf49ba34722297fa1d6bd57 Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Fri, 14 Feb 2025 22:23:21 +0000 Subject: [PATCH 1/2] Add feature to make more thorough reports --- README.md | 17 +++++++++++++++++ src/pytest_nm_releng/plugin.py | 8 ++++++++ tests/test_helpers.py | 35 +++++++++++++++++++++++++++++++++- tests/test_nm_releng.py | 22 +++++++++++++++++++-- 4 files changed, 79 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c87770b..04a19d2 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,23 @@ pytest [...] # `test-results/report-1735941218.338192.xml` will be created ``` +### Thorough JUnit report files + +`pytest-nm-releng` can append some flags that will make the generated JUnit report files more thorough/comprehensive: + +- All output will be included in the reported (including stdout/stderr like logging) +- All typical results/output will be included for passing tests (normally this is only captured for failing/etc. tests) + +> ![NOTE] +> This does _not_ append any flags to actually generate reports. This must be done manually or with the [Dynamically-named JUnit report files](#dynamically-named-junit-report-files) feature. + +To enable this feature, set the `NMRE_JUNIT_FULL` env var to `1`: + +```shell +# example: prefixing a command +NMRE_JUNIT_FULL=1 pytest [...] +``` + ### Code coverage `pytest-nm-releng` can automatically add some code coverage flags as well (requires [pytest-cov]). diff --git a/src/pytest_nm_releng/plugin.py b/src/pytest_nm_releng/plugin.py index f26e231..68f2bad 100644 --- a/src/pytest_nm_releng/plugin.py +++ b/src/pytest_nm_releng/plugin.py @@ -34,6 +34,13 @@ def generate_junit_flags() -> list[str]: return [f"--junit-xml={junitxml_file.as_posix()}"] +def generate_full_junit_flags() -> list[str]: + if os.environ.get("NMRE_JUNIT_FULL", "") == "1": + return ["-o", "junit_logging=all", "-o", "junit_log_passing_tests=True"] + + return [] + + def generate_coverage_flags() -> list[str]: if not (cc_package_name := os.getenv("NMRE_COV_NAME")): return [] @@ -49,5 +56,6 @@ def generate_coverage_flags() -> list[str]: def pytest_load_initial_conftests(early_config, args: list[str], parser): new_args: list[str] = [] new_args.extend(generate_junit_flags()) + new_args.extend(generate_full_junit_flags()) new_args.extend(generate_coverage_flags()) args[:] = [*args, *new_args] diff --git a/tests/test_helpers.py b/tests/test_helpers.py index fda16dd..5491e67 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -18,7 +18,11 @@ import pytest -from pytest_nm_releng.plugin import generate_coverage_flags, generate_junit_flags +from pytest_nm_releng.plugin import ( + generate_coverage_flags, + generate_full_junit_flags, + generate_junit_flags, +) from tests.utils import setenv EnvVarValue = Union[str, None] @@ -51,6 +55,35 @@ def test_generate_coverage_flags_set( assert result == expected_flags +@pytest.mark.parametrize( + "value", + [ + pytest.param("vllm", id="value:vllm"), + pytest.param("", id="empty"), + pytest.param("1", id="1"), + pytest.param("0", id="0"), + pytest.param("2", id="2"), + pytest.param("-1", id="-1"), + pytest.param(None, id="unset"), + ], +) +def test_generate_full_junit_flags(monkeypatch: pytest.MonkeyPatch, value: EnvVarValue): + setenv(monkeypatch, "NMRE_JUNIT_FULL", value) + + if value == "1": + expected_flags = [ + "-o", + "junit_logging=all", + "-o", + "junit_log_passing_tests=True", + ] + else: + expected_flags = [] + + result = generate_full_junit_flags() + assert result == expected_flags + + @pytest.mark.parametrize( ("env_junit_base", "env_junit_prefix"), [ diff --git a/tests/test_nm_releng.py b/tests/test_nm_releng.py index 1010c3f..9d6db5e 100644 --- a/tests/test_nm_releng.py +++ b/tests/test_nm_releng.py @@ -51,6 +51,19 @@ def test_plugin_adds_junit_args( assert actual.startswith("results") +def test_plugin_adds_full_junit_args( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +): + """Verify the plugin adds the expected junit args""" + + monkeypatch.setenv("NMRE_JUNIT_FULL", "1") + + cf = pytester.parseconfigure() + actual = cf.getoption("-o", None) + assert actual is not None + assert actual == ["junit_logging=all", "junit_log_passing_tests=True"] + + @pytest.mark.skipif(not _pytest_cov_installed, reason="pytest-cov is required") def test_plugin_adds_coverage_args( pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch @@ -72,10 +85,15 @@ def test_plugin_adds_all_args( """Verify the plugin adds the expected coverage args""" monkeypatch.setenv("NMRE_JUNIT_BASE", "results") + monkeypatch.setenv("NMRE_JUNIT_FULL", "1") monkeypatch.setenv("NMRE_COV_NAME", "vllm") cf = pytester.parseconfigure() - assert cf.getoption("--junit-xml", None).startswith("results") - assert "vllm" in cf.getoption("--cov", None) + assert cf.getoption("--junit-xml", "").startswith("results") + assert "vllm" in cf.getoption("--cov", []) + assert cf.getoption("-o", None) == [ + "junit_logging=all", + "junit_log_passing_tests=True", + ] assert cf.getoption("--cov-append") is True From 075bf1972b6cd76241020d9b6f9f1b81ab6dedd2 Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Fri, 14 Feb 2025 22:23:34 +0000 Subject: [PATCH 2/2] Update tox test env to run all tests --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 09f75ba..f0befbc 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,8 @@ envlist = py39,py310,py311,py312 [testenv] deps = - pytest >= 8 + pytest ~= 8.3 + pytest-cov ~= 6.0 commands = pytest {posargs:tests}