From ca739809bcfd13bd594a9b3f64efcc24b7772942 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Tue, 15 Nov 2022 14:05:40 +0200 Subject: [PATCH 01/32] update(docs): increase version --- universum/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/universum/__init__.py b/universum/__init__.py index fe70ce34..97c18b90 100644 --- a/universum/__init__.py +++ b/universum/__init__.py @@ -1,2 +1,2 @@ __title__ = "Universum" -__version__ = "0.19.14" +__version__ = "0.19.15" From 6592c2efebef3c4c7fb1d68f0c7b2fb274d09f8e Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Tue, 27 Dec 2022 18:40:24 +0200 Subject: [PATCH 02/32] add(github): enable actions workflow --- .github/workflows/precommit-check.yml | 38 +++++++++++++++++++++++++++ .universum.py | 3 ++- 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/precommit-check.yml diff --git a/.github/workflows/precommit-check.yml b/.github/workflows/precommit-check.yml new file mode 100644 index 00000000..bfa7d698 --- /dev/null +++ b/.github/workflows/precommit-check.yml @@ -0,0 +1,38 @@ +name: Universum check +on: push + +jobs: + universum_check: + name: Universum check + runs-on: ubuntu-latest + + steps: + - name: Setup python 3.8 + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Install dependency + run: pip install universum[test] + + - name: Universum + run: + python -u -m universum + --vcs-type=git + --git-repo $GITHUB_SERVER_URL/$GITHUB_REPOSITORY + --git-refspec $GITHUB_REF_NAME + --no-archive + --no-diff + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v1 + if: always() + with: + files: artifacts/*.xml + + - name: Collect artifacts + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: artifacts + path: artifacts diff --git a/.universum.py b/.universum.py index 71dc3b24..e3c3eda8 100644 --- a/.universum.py +++ b/.universum.py @@ -20,12 +20,13 @@ def pip_install(module_name): configs = Variations([Step(name="Create virtual environment", command=[python, "-m", "venv", env_name]), Step(name="Update Docker images", command=run_virtual("make images")), - Step(name="Install Universum for tests", artifacts="junit_results.xml", + Step(name="Install Universum for tests", command=run_virtual(pip_install(".[test]"))), Step(name="Make", artifacts="doc/_build", command=run_virtual("make")), Step(name="Make tests", artifacts="htmlcov", command=run_virtual("export LANG=en_US.UTF-8; make test")), + Step(name="Collect test results", artifacts="junit_results.xml"), Step(name="Run static pylint", code_report=True, command=run_virtual(f"{python} -m universum.analyzers.pylint " From b7e5e913a351558be54326c46b55490cfebe7454 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Wed, 11 Jan 2023 14:39:54 +0200 Subject: [PATCH 03/32] test(artifact_collector): basic tests for the set_and_clean_artifacts() logic (#753) --- tests/test_preprocess_artifacts.py | 71 ++++++++++++++++++++++++++++++ tests/utils.py | 3 +- 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 tests/test_preprocess_artifacts.py diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py new file mode 100644 index 00000000..8ab25d57 --- /dev/null +++ b/tests/test_preprocess_artifacts.py @@ -0,0 +1,71 @@ +import inspect +import os +import pytest + +from .utils import LocalTestEnvironment + + +@pytest.fixture() +def test_env(tmpdir): + yield ArtifactsTestEnvironment(tmpdir) + + +def test_no_artifact_prebuild_clean(test_env): + test_env.write_config_file(artifact_prebuild_clean=True) + test_env.run() + test_env.check_step_artifact_present() + + +def test_no_artifact_no_prebuild_clean(test_env): + test_env.write_config_file(artifact_prebuild_clean=False) + test_env.run() + test_env.check_step_artifact_present() + + +def test_existing_artifact_prebuild_clean(test_env): + test_env.write_config_file(artifact_prebuild_clean=True) + test_env.create_artifact_file() + test_env.run() + test_env.check_step_artifact_present() + + +def test_existing_artifact_no_prebuild_clean(test_env, stdout_checker): + test_env.write_config_file(artifact_prebuild_clean=False) + test_env.create_artifact_file() + test_env.run(expect_failure=True) + stdout_checker.assert_has_calls_with_param(f"already exist in '/.*' directory", is_regexp=True) + test_env.check_step_artifact_absent() + + +class ArtifactsTestEnvironment(LocalTestEnvironment): + + def __init__(self, tmpdir): + super().__init__(tmpdir, "main") + self.artifact_name = "artifact" + self.artifact_path = self.artifact_dir.join(self.artifact_name) + self.artifact_content = "artifact content" + + def write_config_file(self, artifact_prebuild_clean): + config = inspect.cleandoc(f""" + from universum.configuration_support import Configuration, Step + step = Step(name='Step', + command=['bash', '-c', 'echo "{self.artifact_content}" > {self.artifact_name}'], + artifacts='{self.artifact_name}', + artifact_prebuild_clean={artifact_prebuild_clean}) + configs = Configuration([step]) + """) + self.configs_file.write_text(config, "utf-8") + + def check_step_artifact_present(self): + assert os.path.exists(self.artifact_path) + with open(self.artifact_path) as f: + content = f.read().replace("\n", "") + assert content == self.artifact_content + + def check_step_artifact_absent(self): + assert not os.path.exists(self.artifact_path) + + def create_artifact_file(self): + precreated_artifact = self.src_dir.join(self.artifact_name) + with open(precreated_artifact, "w") as f: + f.write("pre-created artifact content") diff --git a/tests/utils.py b/tests/utils.py index 1fcc08ab..77e8efc4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -257,4 +257,5 @@ def __init__(self, directory, test_type): if test_type != "nonci": self.settings.Vcs.type = "none" if test_type == "main": - self.settings.LocalMainVcs.source_dir = str(directory.ensure_dir('project_sources')) + self.src_dir = directory.ensure_dir('project_sources') + self.settings.LocalMainVcs.source_dir = str(self.src_dir) From 994f21804a96ddee6ec45a112bb5d8ab499f8327 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Fri, 20 Jan 2023 16:24:50 +0200 Subject: [PATCH 04/32] fix(checks): MyPy and Pylint warnings Also turn on failing on failed steps for better GhA reporting --- .github/workflows/precommit-check.yml | 1 + tests/test_argument_check.py | 4 ++-- tests/test_git_poll.py | 4 +++- tests/test_integration.py | 1 - tests/test_preprocess_artifacts.py | 8 +++++--- tests/utils.py | 8 ++++---- universum/analyzers/utils.py | 2 +- universum/configuration_support.py | 2 +- universum/lib/utils.py | 4 ++-- universum/modules/artifact_collector.py | 6 ++---- universum/modules/reporter.py | 10 +++++----- universum/modules/vcs/perforce_vcs.py | 12 ++++++------ universum/modules/vcs/vcs.py | 2 +- 13 files changed, 33 insertions(+), 31 deletions(-) diff --git a/.github/workflows/precommit-check.yml b/.github/workflows/precommit-check.yml index bfa7d698..1700d7eb 100644 --- a/.github/workflows/precommit-check.yml +++ b/.github/workflows/precommit-check.yml @@ -18,6 +18,7 @@ jobs: - name: Universum run: python -u -m universum + --fail-unsuccessful --vcs-type=git --git-repo $GITHUB_SERVER_URL/$GITHUB_REPOSITORY --git-refspec $GITHUB_REF_NAME diff --git a/tests/test_argument_check.py b/tests/test_argument_check.py index 49344cf8..05974c80 100644 --- a/tests/test_argument_check.py +++ b/tests/test_argument_check.py @@ -1,4 +1,4 @@ -from typing import Union, List +from typing import Union, List, Optional import pytest from universum import __main__ @@ -101,7 +101,7 @@ def assert_incorrect_parameter(settings: ModuleNamespace, *args): def param(test_type: str, module: str, field: str, - vcs_type: Union[str, List[str]] = "*", error_match: str = None) -> None: + vcs_type: Union[str, List[str]] = "*", error_match: Optional[str] = None) -> None: if isinstance(vcs_type, list): for specific_vcs_type in vcs_type: diff --git a/tests/test_git_poll.py b/tests/test_git_poll.py index 437014fe..219849d1 100644 --- a/tests/test_git_poll.py +++ b/tests/test_git_poll.py @@ -1,5 +1,6 @@ # pylint: disable = redefined-outer-name +from typing import Optional import py import pytest @@ -13,7 +14,8 @@ def git_poll_environment(git_client: GitClient, tmpdir: py.path.local): yield GitTestEnvironment(git_client, tmpdir, test_type="poll") -def make_branch_with_changes(git_server: GitServer, branch_name: str, commits_number: int, branch_from: str = None): +def make_branch_with_changes(git_server: GitServer, branch_name: str, commits_number: int, + branch_from: Optional[str] = None): """ Creates a branch from the current or specified (by name) and adds passed commits number. diff --git a/tests/test_integration.py b/tests/test_integration.py index 6b93202a..d7a3749a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -379,4 +379,3 @@ def test_exit_code(local_sources: LocalSources, tmpdir: py.path.local): "-fsd", str(local_sources.root_directory), "-cfg", str(config_file)]) as process: assert process.wait() == 1 - diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index 8ab25d57..35cbbf49 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -1,3 +1,5 @@ +# pylint: disable = redefined-outer-name + import inspect import os import pytest @@ -33,7 +35,7 @@ def test_existing_artifact_no_prebuild_clean(test_env, stdout_checker): test_env.write_config_file(artifact_prebuild_clean=False) test_env.create_artifact_file() test_env.run(expect_failure=True) - stdout_checker.assert_has_calls_with_param(f"already exist in '/.*' directory", is_regexp=True) + stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) test_env.check_step_artifact_absent() @@ -58,7 +60,7 @@ def write_config_file(self, artifact_prebuild_clean): def check_step_artifact_present(self): assert os.path.exists(self.artifact_path) - with open(self.artifact_path) as f: + with open(self.artifact_path, encoding="utf-8") as f: content = f.read().replace("\n", "") assert content == self.artifact_content @@ -67,5 +69,5 @@ def check_step_artifact_absent(self): def create_artifact_file(self): precreated_artifact = self.src_dir.join(self.artifact_name) - with open(precreated_artifact, "w") as f: + with open(precreated_artifact, "w", encoding="utf-8") as f: f.write("pre-created artifact content") diff --git a/tests/utils.py b/tests/utils.py index 77e8efc4..52c6289c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -109,20 +109,20 @@ def create_empty_settings(test_type: str) -> ModuleNamespace: class BaseVcsClient: - def __init__(self): + def __init__(self) -> None: self.root_directory: py.path.local self.repo_file: py.path.local def get_last_change(self): pass - def file_present(self, file_path: str) -> bool: + def file_present(self, file_path: str) -> bool: # type: ignore pass - def text_in_file(self, text: str, file_path: str) -> bool: + def text_in_file(self, text: str, file_path: str) -> bool: # type: ignore pass - def make_a_change(self) -> str: + def make_a_change(self) -> str: # type: ignore pass diff --git a/universum/analyzers/utils.py b/universum/analyzers/utils.py index 76e70175..a6bfd452 100644 --- a/universum/analyzers/utils.py +++ b/universum/analyzers/utils.py @@ -137,7 +137,7 @@ def add_python_version_argument(parser: argparse.ArgumentParser) -> None: "'python3.7 -m pylint <...>'") -def report_to_file(issues: List[ReportData], json_file: str = None) -> None: +def report_to_file(issues: List[ReportData], json_file: Optional[str] = None) -> None: issues_json = json.dumps(issues, indent=4) if json_file: with open(json_file, "w", encoding="utf-8") as f: diff --git a/universum/configuration_support.py b/universum/configuration_support.py index 3bdb8d11..c911d560 100644 --- a/universum/configuration_support.py +++ b/universum/configuration_support.py @@ -672,7 +672,7 @@ def dump(self, produce_string_command: bool = True) -> str: return result def filter(self, checker: Callable[[Step], bool], - parent: Step = None) -> 'Configuration': + parent: Optional[Step] = None) -> 'Configuration': """ This function is supposed to be called from main script, not configuration file. It uses provided `checker` to find all the configurations that pass the check, diff --git a/universum/lib/utils.py b/universum/lib/utils.py index 02154d7f..b924a6e9 100644 --- a/universum/lib/utils.py +++ b/universum/lib/utils.py @@ -104,10 +104,10 @@ def format_traceback(exc: Exception, trace: Optional[TracebackType]) -> str: return tb_text -def catch_exception(exception_name: str, ignore_if: str = None) -> DecoratorT: +def catch_exception(exception_name: str, ignore_if: Optional[str] = None) -> DecoratorT: def decorated_function(function): def function_to_run(*args, **kwargs): - result: ReturnT = None + result: ReturnT = None # type: ignore try: result = function(*args, **kwargs) return result diff --git a/universum/modules/artifact_collector.py b/universum/modules/artifact_collector.py index e6a5c094..f37d7df9 100644 --- a/universum/modules/artifact_collector.py +++ b/universum/modules/artifact_collector.py @@ -1,6 +1,4 @@ import codecs -import distutils -from distutils import dir_util, errors import os import shutil import zipfile @@ -201,11 +199,11 @@ def move_artifact(self, path, is_report=False): # Single file archiving is not implemented at the moment pass try: - distutils.dir_util.copy_tree(matching_path, destination) + shutil.copytree(matching_path, destination) if is_report: text = "'" + artifact_name + "' is not a file and cannot be reported as an artifact" self.out.log(text) - except distutils.errors.DistutilsFileError: + except NotADirectoryError: shutil.copyfile(matching_path, destination) if is_report: artifact_path = self.automation_server.artifact_path(self.artifact_dir, artifact_name) diff --git a/universum/modules/reporter.py b/universum/modules/reporter.py index 5c150563..0d3c95c2 100644 --- a/universum/modules/reporter.py +++ b/universum/modules/reporter.py @@ -57,12 +57,12 @@ def define_arguments(argument_parser): parser.add_argument("--fail-unsuccessful", "-rfu", action="store_true", dest="fail_unsuccessful", help="Return non-zero exit code if any step failed") - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self.observers = [] - self.report_initialized = False - self.blocks_to_report = [] - self.artifacts_to_report = [] + self.observers: List = [] + self.report_initialized: bool = False + self.blocks_to_report: List = [] + self.artifacts_to_report: List = [] self.code_report_comments: Dict[str, List[ReportMessage]] = defaultdict(list) self.automation_server = self.automation_server_factory() diff --git a/universum/modules/vcs/perforce_vcs.py b/universum/modules/vcs/perforce_vcs.py index 8974b3d7..e0fc8618 100644 --- a/universum/modules/vcs/perforce_vcs.py +++ b/universum/modules/vcs/perforce_vcs.py @@ -299,7 +299,7 @@ def define_arguments(argument_parser): help="**Revert all vcs within '--p4-client' and delete the workspace.** " "Mandatory for CI environment, otherwise use with caution") - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.check_required_option("client", """ @@ -318,11 +318,11 @@ def __init__(self, *args, **kwargs): self.client_name = self.settings.client self.client_root = self.settings.project_root - self.sync_cls = [] - self.shelve_cls = [] - self.depots = [] - self.client_view = [] - self.mappings_dict = {} + self.sync_cls: List = [] + self.shelve_cls: List = [] + self.depots: List = [] + self.client_view: List = [] + self.mappings_dict: Dict = {} self.unshelved_files: List[Dict[str, str]] = [] self.diff_in_files: List[Tuple[Optional[str], Optional[str], Optional[str]]] = [] diff --git a/universum/modules/vcs/vcs.py b/universum/modules/vcs/vcs.py index cb31fd08..9884bd6f 100644 --- a/universum/modules/vcs/vcs.py +++ b/universum/modules/vcs/vcs.py @@ -20,7 +20,7 @@ ] -def create_vcs(class_type: str = None) -> Type[ProjectDirectory]: +def create_vcs(class_type: Optional[str] = None) -> Type[ProjectDirectory]: driver_factory_class: Union[ Dict[str, Type[base_vcs.BasePollVcs]], Dict[str, Type[base_vcs.BaseSubmitVcs]], From bf8521028a1142dcf8135d27459c43c44744048c Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Mon, 23 Jan 2023 09:39:01 +0200 Subject: [PATCH 05/32] test(artifact_collector): add typing to test_preprocess_artifacts.py (#754) --- tests/test_preprocess_artifacts.py | 83 ++++++++++++++++-------------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index 35cbbf49..fe597c7c 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -2,72 +2,77 @@ import inspect import os +import pathlib +from typing import Generator + import pytest from .utils import LocalTestEnvironment +from .conftest import FuzzyCallChecker + + +class ArtifactsTestEnvironment(LocalTestEnvironment): + + def __init__(self, tmpdir: pathlib.Path) -> None: + super().__init__(tmpdir, "main") + self.artifact_name: str = "artifact" + self.artifact_path: pathlib.Path = self.artifact_dir.join(self.artifact_name) + self.artifact_content: str = "artifact content" + + def write_config_file(self, artifact_prebuild_clean: bool) -> None: + config: str = inspect.cleandoc(f""" + from universum.configuration_support import Configuration, Step + step = Step(name='Step', + command=['bash', '-c', 'echo "{self.artifact_content}" > {self.artifact_name}'], + artifacts='{self.artifact_name}', + artifact_prebuild_clean={artifact_prebuild_clean}) + configs = Configuration([step]) + """) + self.configs_file.write_text(config, "utf-8") + + def check_step_artifact_present(self) -> None: + assert os.path.exists(self.artifact_path) + with open(self.artifact_path, encoding="utf-8") as f: + content: str = f.read().replace("\n", "") + assert content == self.artifact_content + + def check_step_artifact_absent(self) -> None: + assert not os.path.exists(self.artifact_path) + + def create_artifact_file(self) -> None: + precreated_artifact: pathlib.Path = self.src_dir.join(self.artifact_name) + with open(precreated_artifact, "w", encoding="utf-8") as f: + f.write("pre-created artifact content") @pytest.fixture() -def test_env(tmpdir): +def test_env(tmpdir: pathlib.Path) -> Generator[ArtifactsTestEnvironment, None, None]: yield ArtifactsTestEnvironment(tmpdir) -def test_no_artifact_prebuild_clean(test_env): +def test_no_artifact_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: test_env.write_config_file(artifact_prebuild_clean=True) test_env.run() test_env.check_step_artifact_present() -def test_no_artifact_no_prebuild_clean(test_env): +def test_no_artifact_no_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: test_env.write_config_file(artifact_prebuild_clean=False) test_env.run() test_env.check_step_artifact_present() -def test_existing_artifact_prebuild_clean(test_env): +def test_existing_artifact_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: test_env.write_config_file(artifact_prebuild_clean=True) test_env.create_artifact_file() test_env.run() test_env.check_step_artifact_present() -def test_existing_artifact_no_prebuild_clean(test_env, stdout_checker): +def test_existing_artifact_no_prebuild_clean(test_env: ArtifactsTestEnvironment, + stdout_checker: FuzzyCallChecker) -> None: test_env.write_config_file(artifact_prebuild_clean=False) test_env.create_artifact_file() test_env.run(expect_failure=True) stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) test_env.check_step_artifact_absent() - - -class ArtifactsTestEnvironment(LocalTestEnvironment): - - def __init__(self, tmpdir): - super().__init__(tmpdir, "main") - self.artifact_name = "artifact" - self.artifact_path = self.artifact_dir.join(self.artifact_name) - self.artifact_content = "artifact content" - - def write_config_file(self, artifact_prebuild_clean): - config = inspect.cleandoc(f""" - from universum.configuration_support import Configuration, Step - step = Step(name='Step', - command=['bash', '-c', 'echo "{self.artifact_content}" > {self.artifact_name}'], - artifacts='{self.artifact_name}', - artifact_prebuild_clean={artifact_prebuild_clean}) - configs = Configuration([step]) - """) - self.configs_file.write_text(config, "utf-8") - - def check_step_artifact_present(self): - assert os.path.exists(self.artifact_path) - with open(self.artifact_path, encoding="utf-8") as f: - content = f.read().replace("\n", "") - assert content == self.artifact_content - - def check_step_artifact_absent(self): - assert not os.path.exists(self.artifact_path) - - def create_artifact_file(self): - precreated_artifact = self.src_dir.join(self.artifact_name) - with open(precreated_artifact, "w", encoding="utf-8") as f: - f.write("pre-created artifact content") From 91afad6be101e9f657c20c7834be11dc4c27ddf2 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Tue, 31 Jan 2023 12:14:16 +0200 Subject: [PATCH 06/32] test: use `tmp_path` fixture instead of `tmpdir` (#757) --- tests/deployment_utils.py | 20 +-- tests/git_utils.py | 36 +++--- tests/perforce_utils.py | 37 +++--- tests/test_api.py | 14 +-- tests/test_code_report.py | 42 +++---- tests/test_create_config.py | 6 +- tests/test_deployment.py | 10 +- tests/test_git_poll.py | 6 +- tests/test_github_handler.py | 8 +- tests/test_html_output.py | 34 ++--- tests/test_integration.py | 46 +++---- tests/test_nonci.py | 2 +- tests/test_p4_exception_handling.py | 12 +- tests/test_p4_revert_unshelved.py | 27 ++-- tests/test_p4_submit.py | 8 +- tests/test_poll.py | 28 ++--- tests/test_preprocess_artifacts.py | 12 +- tests/test_regression.py | 34 ++--- tests/test_report.py | 8 +- tests/test_run_steps_filter.py | 28 ++--- tests/test_submit.py | 184 +++++++++++++++------------- tests/test_unicode.py | 28 +++-- tests/utils.py | 29 +++-- universum/analyzers/uncrustify.py | 28 ++--- universum/config_creator.py | 4 +- 25 files changed, 361 insertions(+), 330 deletions(-) diff --git a/tests/deployment_utils.py b/tests/deployment_utils.py index 4dee2a32..8d080315 100644 --- a/tests/deployment_utils.py +++ b/tests/deployment_utils.py @@ -2,10 +2,11 @@ import getpass import os +import shutil +import pathlib from pwd import getpwnam import docker -import py import pytest from requests.exceptions import ReadTimeout @@ -154,26 +155,27 @@ def clean_execution_environment(request): class LocalSources(utils.BaseVcsClient): - def __init__(self, root_directory: py.path.local, repo_file: py.path.local): + def __init__(self, root_directory: pathlib.Path, repo_file: pathlib.Path): super().__init__() self.root_directory = root_directory self.repo_file = repo_file @pytest.fixture() -def local_sources(tmpdir: py.path.local): +def local_sources(tmp_path: pathlib.Path): if utils.reuse_docker_containers(): - source_dir = py.path.local(".work") + source_dir = pathlib.Path(".work") try: - source_dir.remove(rec=1, ignore_errors=True) + shutil.rmtree(source_dir, ignore_errors=True) except OSError: pass - source_dir.ensure(dir=True) + source_dir.mkdir() else: - source_dir = tmpdir.mkdir("project_sources") - local_file = source_dir.join("readme.txt") - local_file.write("This is a an empty file") + source_dir = tmp_path / "project_sources" + source_dir.mkdir() + local_file = source_dir / "readme.txt" + local_file.write_text("This is a an empty file", encoding="utf-8") yield LocalSources(root_directory=source_dir, repo_file=local_file) diff --git a/tests/git_utils.py b/tests/git_utils.py index a4bb6cb5..2d02a2d5 100644 --- a/tests/git_utils.py +++ b/tests/git_utils.py @@ -1,11 +1,11 @@ # pylint: disable = redefined-outer-name import os +import pathlib from time import sleep import git from git.remote import RemoteProgress -import py import pytest import sh @@ -13,11 +13,11 @@ class GitServer: - def __init__(self, working_directory: py.path.local, branch_name: str): + def __init__(self, working_directory: pathlib.Path, branch_name: str): self.target_branch: str = branch_name self.target_file: str = "readme.txt" - self._working_directory: py.path.local = working_directory + self._working_directory: pathlib.Path = working_directory self._repo: git.Repo = git.Repo.init(working_directory) self._repo.daemon_export = True self._daemon_started: bool = False @@ -42,8 +42,8 @@ def std_redirect(line): with self._repo.config_writer() as configurator: configurator.set_value("user", "name", "Testing user") configurator.set_value("user", "email", "some@email.com") - self._file: py.path.local = self._working_directory.join(self.target_file) - self._file.write("") + self._file: pathlib.Path = self._working_directory / self.target_file + self._file.write_text("") self._repo.index.add([str(self._file)]) self._repo.index.commit("initial commit") @@ -57,7 +57,7 @@ def std_redirect(line): def make_a_change(self) -> str: self._branch.checkout() - self._file.write("One more line\n") + self._file.write_text("One more line\n") self._commit_count += 1 self._repo.index.add([str(self._file)]) @@ -68,8 +68,8 @@ def make_a_change(self) -> str: def commit_new_file(self) -> str: """ Make a mergeble commit """ self._commit_count += 1 - test_file = self._working_directory.join(f"test{self._commit_count}.txt") - test_file.write(f"Commit number #{self._commit_count}") + test_file = self._working_directory / f"test{self._commit_count}.txt" + test_file.write_text(f"Commit number #{self._commit_count}") self._repo.index.add([str(test_file)]) return str(self._repo.index.commit(f"Add file {self._commit_count}")) @@ -104,8 +104,9 @@ def exit(self) -> None: @pytest.fixture() -def git_server(tmpdir: py.path.local): - directory = tmpdir.mkdir("server") +def git_server(tmp_path: pathlib.Path): + directory = tmp_path / "server" + directory.mkdir() server = GitServer(directory, "testing") try: yield server @@ -114,7 +115,7 @@ def git_server(tmpdir: py.path.local): class GitClient(utils.BaseVcsClient): - def __init__(self, git_server: GitServer, directory: py.path.local): + def __init__(self, git_server: GitServer, directory: pathlib.Path): super().__init__() class Progress(RemoteProgress): @@ -123,9 +124,10 @@ def line_dropped(self, line): self.server: GitServer = git_server self.logger: Progress = Progress() - self.root_directory: py.path.local = directory.mkdir("client") + self.root_directory: pathlib.Path = directory / "client" + self.root_directory.mkdir() self.repo: git.Repo = git.Repo.clone_from(git_server.url, self.root_directory) - self.repo_file = self.root_directory.join(git_server.target_file) + self.repo_file = self.root_directory / git_server.target_file def get_last_change(self) -> str: changes = self.repo.git.log("origin/" + self.server.target_branch, pretty="oneline", max_count=1) @@ -144,13 +146,13 @@ def make_a_change(self) -> str: @pytest.fixture() -def git_client(git_server: GitServer, tmpdir: py.path.local): - yield GitClient(git_server, tmpdir) +def git_client(git_server: GitServer, tmp_path: pathlib.Path): + yield GitClient(git_server, tmp_path) class GitTestEnvironment(utils.BaseTestEnvironment): - def __init__(self, client: GitClient, directory: py.path.local, test_type: str): - db_file = directory.join("gitpoll.json") + def __init__(self, client: GitClient, directory: pathlib.Path, test_type: str): + db_file = directory / "gitpoll.json" super().__init__(client, directory, test_type, str(db_file)) self.vcs_client: GitClient diff --git a/tests/perforce_utils.py b/tests/perforce_utils.py index 53e628db..a570fc7b 100644 --- a/tests/perforce_utils.py +++ b/tests/perforce_utils.py @@ -1,9 +1,9 @@ # pylint: disable = redefined-outer-name, too-many-locals import time +import pathlib import docker -import py import pytest from P4 import P4, P4Exception from requests.exceptions import ReadTimeout @@ -177,12 +177,13 @@ def perforce_connection(request, docker_perforce: PerfoceDockerContainer): class PerforceWorkspace(utils.BaseVcsClient): - def __init__(self, connection: PerforceConnection, directory: py.path.local): + def __init__(self, connection: PerforceConnection, directory: pathlib.Path): super().__init__() - self.root_directory = directory.mkdir("workspace") - self.repo_file = self.root_directory.join("writeable_file.txt") + self.root_directory = directory / "workspace" + self.root_directory.mkdir() + self.repo_file = self.root_directory / "writeable_file.txt" - self.nonwritable_file: py.path.local = self.root_directory.join("usual_file.txt") + self.nonwritable_file: pathlib.Path = self.root_directory / "usual_file.txt" self.server: PerfoceDockerContainer = connection.server self.client_created: bool = False @@ -202,10 +203,10 @@ def setup(self) -> None: self.p4.run("add", str(self.nonwritable_file)) self.p4.run("edit", str(self.nonwritable_file)) - self.nonwritable_file.write("File " + str(self.nonwritable_file) + " has no special modifiers") + self.nonwritable_file.write_text("File " + str(self.nonwritable_file) + " has no special modifiers") self.p4.run("add", "-t", "+w", str(self.repo_file)) - self.repo_file.write("File " + str(self.repo_file) + " is always writable") + self.repo_file.write_text("File " + str(self.repo_file) + " is always writable") change = self.p4.run_change("-o")[0] change["Description"] = "Test submit" @@ -225,9 +226,9 @@ def setup(self) -> None: ]} self.p4.save_triggers(triggers) - def create_file(self, file_name: str) -> py.path.local: - p4_new_file = self.root_directory.join(file_name) - p4_new_file.write("This is unchanged line 1\nThis is unchanged line 2") + def create_file(self, file_name: str) -> pathlib.Path: + p4_new_file = self.root_directory / file_name + p4_new_file.write_text("This is unchanged line 1\nThis is unchanged line 2") self.p4.run("add", str(p4_new_file)) change = self.p4.run_change("-o")[0] @@ -241,14 +242,14 @@ def delete_file(self, file_name: str) -> None: change["Description"] = "Delete created file" self.p4.run_submit(change) - def shelve_file(self, file: py.path.local, content: str, shelve_cl=None) -> str: + def shelve_file(self, file: pathlib.Path, content: str, shelve_cl=None) -> str: if not shelve_cl: change = self.p4.fetch_change() change["Description"] = "This is a shelved CL" shelve_cl = self.p4.save_change(change)[0].split()[1] self.p4.run_edit("-c", shelve_cl, str(file)) - file.write(content) + file.write_text(content) self.p4.run_shelve("-fc", shelve_cl) self.p4.run_revert("-c", shelve_cl, str(file)) return shelve_cl @@ -274,7 +275,7 @@ def file_present(self, file_path: str) -> bool: def make_a_change(self) -> str: tmpfile = self.repo_file self.p4.run("edit", str(tmpfile)) - tmpfile.write("Change #1 " + str(tmpfile)) + tmpfile.write_text("Change #1 " + str(tmpfile)) change = self.p4.run_change("-o")[0] change["Description"] = "Test submit #1" @@ -293,8 +294,8 @@ def cleanup(self) -> None: @pytest.fixture() -def perforce_workspace(request, perforce_connection: PerforceConnection, tmpdir: py.path.local): - workspace = PerforceWorkspace(perforce_connection, tmpdir) +def perforce_workspace(request, perforce_connection: PerforceConnection, tmp_path: pathlib.Path): + workspace = PerforceWorkspace(perforce_connection, tmp_path) try: workspace.setup() yield workspace @@ -303,8 +304,8 @@ def perforce_workspace(request, perforce_connection: PerforceConnection, tmpdir: class P4TestEnvironment(utils.BaseTestEnvironment): - def __init__(self, perforce_workspace: PerforceWorkspace, directory: py.path.local, test_type: str): - db_file = directory.join("p4poll.json") + def __init__(self, perforce_workspace: PerforceWorkspace, directory: pathlib.Path, test_type: str): + db_file = directory / "p4poll.json" super().__init__(perforce_workspace, directory, test_type, str(db_file)) self.vcs_client: PerforceWorkspace @@ -332,4 +333,4 @@ def shelve_config(self, config: str) -> None: shelve_cl = self.vcs_client.shelve_file(self.vcs_client.repo_file, config) settings = self.settings settings.PerforceMainVcs.shelve_cls = [shelve_cl] - settings.Launcher.config_path = self.vcs_client.repo_file.basename + settings.Launcher.config_path = self.vcs_client.repo_file.name diff --git a/tests/test_api.py b/tests/test_api.py index 6b48624b..b41de1b2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -24,8 +24,8 @@ def test_p4_file_diff(docker_main_with_vcs: UniversumRunner): p4_file = docker_main_with_vcs.perforce.repo_file p4.run_edit(p4_file) - p4_file.write("This line is added to the file.\n") - p4.run_move(p4_file, p4_directory.join("some_new_file_name.txt")) + p4_file.write_text("This line is added to the file.\n") + p4.run_move(p4_file, p4_directory / "some_new_file_name.txt") change = p4.fetch_change() change["Description"] = "Rename basic config" shelve_cl = p4.save_change(change)[0].split()[1] @@ -49,8 +49,8 @@ def test_multiple_p4_file_diff(docker_main_with_vcs: UniversumRunner): p4_directory = docker_main_with_vcs.perforce.root_directory for index in range(0, 10000): - new_file = p4_directory.join(f"new_file_{index}.txt") - new_file.write(f"This is file #{index}\n") + new_file = p4_directory / f"new_file_{index}.txt" + new_file.write_text(f"This is file #{index}\n") p4.run_add(new_file) change = p4.fetch_change() change["Description"] = "Add 10000 files" @@ -77,7 +77,7 @@ def test_git_file_diff(docker_main_with_vcs: UniversumRunner): repo.git.checkout(server.target_branch) repo.git.checkout("new_testing_branch", b=True) - repo.git.mv(git_file, git_directory.join("some_new_file_name.txt")) + repo.git.mv(git_file, git_directory / "some_new_file_name.txt") change = repo.index.commit("Special commit for testing") repo.remotes.origin.push(progress=logger, all=True) @@ -104,8 +104,8 @@ def test_multiple_git_file_diff(docker_main_with_vcs: UniversumRunner): repo.git.checkout("new_testing_branch", b=True) files = [] for index in range(0, 10000): - new_file = git_directory.join(f"new_file_{index}.txt") - new_file.write(f"This is file #{index}\n") + new_file = git_directory / f"new_file_{index}.txt" + new_file.write_text(f"This is file #{index}\n") files.append(str(new_file)) repo.index.add(files) change = repo.index.commit("Special commit for testing") diff --git a/tests/test_code_report.py b/tests/test_code_report.py index 9383afd9..9e57306c 100644 --- a/tests/test_code_report.py +++ b/tests/test_code_report.py @@ -1,9 +1,9 @@ import inspect import os import re +import pathlib from typing import List -import py import pytest from . import utils @@ -157,7 +157,7 @@ def test_code_report_direct_log(runner_with_analyzers: UniversumRunner, tested_c for idx, tested_content in enumerate(tested_contents): prelim_report = "report_file_" + str(idx) full_report = "${CODE_REPORT_FILE}" - runner_with_analyzers.local.root_directory.join(prelim_report).write(tested_content) + (runner_with_analyzers.local.root_directory / prelim_report).write_text(tested_content) config.add_cmd("Report " + str(idx), f"[\"bash\", \"-c\", \"cat './{prelim_report}' >> '{full_report}'\"]", step_config) log = runner_with_analyzers.run(config.finalize()) @@ -187,13 +187,13 @@ def test_code_report_log(runner_with_analyzers: UniversumRunner, analyzers, extr "--result-file", "${CODE_REPORT_FILE}", "--files", "source_file", ] - runner_with_analyzers.local.root_directory.join("source_file").write(tested_content) + (runner_with_analyzers.local.root_directory / "source_file").write_text(tested_content) config = ConfigData() for analyzer in analyzers: args = common_args + extra_args if analyzer == 'uncrustify': args += ["--cfg-file", "cfg"] - runner_with_analyzers.local.root_directory.join("cfg").write(config_uncrustify) + (runner_with_analyzers.local.root_directory / "cfg").write_text(config_uncrustify) config.add_analyzer(analyzer, args) log = runner_with_analyzers.run(config.finalize()) @@ -241,8 +241,8 @@ def test_analyzer_python_version_params(runner_with_analyzers: UniversumRunner, "Target and source folders for uncrustify are not allowed to match"], ]) def test_analyzer_specific_params(runner_with_analyzers: UniversumRunner, analyzer, arg_set, expected_log): - source_file = runner_with_analyzers.local.root_directory.join("source_file") - source_file.write(source_code_python) + source_file = runner_with_analyzers.local.root_directory / "source_file" + source_file.write_text(source_code_python) log = runner_with_analyzers.run(ConfigData().add_analyzer(analyzer, arg_set).finalize()) assert re.findall(fr'Run {analyzer} - [^\n]*Failed', log), f"'{analyzer}' info is not found in '{log}'" @@ -261,9 +261,9 @@ def test_analyzer_specific_params(runner_with_analyzers: UniversumRunner, analyz def test_uncrustify_file_diff(runner_with_analyzers: UniversumRunner, extra_args, tested_content, expected_success, expected_artifact): root = runner_with_analyzers.local.root_directory - source_file = root.join("source_file") - source_file.write(tested_content) - root.join("cfg").write(config_uncrustify) + source_file = root / "source_file" + source_file.write_text(tested_content) + (root / "cfg").write_text(config_uncrustify) common_args = [ "--result-file", "${CODE_REPORT_FILE}", "--files", "source_file", @@ -281,13 +281,13 @@ def test_uncrustify_file_diff(runner_with_analyzers: UniversumRunner, assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" -def test_code_report_extended_arg_search(tmpdir: py.path.local, stdout_checker: FuzzyCallChecker): - env = utils.LocalTestEnvironment(tmpdir, "main") +def test_code_report_extended_arg_search(tmp_path: pathlib.Path, stdout_checker: FuzzyCallChecker): + env = utils.LocalTestEnvironment(tmp_path, "main") env.settings.Vcs.type = "none" - env.settings.LocalMainVcs.source_dir = str(tmpdir) + env.settings.LocalMainVcs.source_dir = str(tmp_path) - source_file = tmpdir.join("source_file.py") - source_file.write(source_code_python + '\n') + source_file = tmp_path / "source_file.py" + source_file.write_text(source_code_python + '\n') config = f""" from universum.configuration_support import Configuration @@ -297,20 +297,20 @@ def test_code_report_extended_arg_search(tmpdir: py.path.local, stdout_checker: --python-version {python_version()} --files {str(source_file)}'])]) """ - env.configs_file.write(config) + env.configs_file.write_text(config) env.run() stdout_checker.assert_has_calls_with_param(log_fail, is_regexp=True) assert os.path.exists(os.path.join(env.settings.ArtifactCollector.artifact_dir, "Run_static_pylint.json")) -def test_code_report_extended_arg_search_embedded(tmpdir: py.path.local, stdout_checker: FuzzyCallChecker): - env = utils.LocalTestEnvironment(tmpdir, "main") +def test_code_report_extended_arg_search_embedded(tmp_path: pathlib.Path, stdout_checker: FuzzyCallChecker): + env = utils.LocalTestEnvironment(tmp_path, "main") env.settings.Vcs.type = "none" - env.settings.LocalMainVcs.source_dir = str(tmpdir) + env.settings.LocalMainVcs.source_dir = str(tmp_path) - source_file = tmpdir.join("source_file.py") - source_file.write(source_code_python + '\n') + source_file = tmp_path / "source_file.py" + source_file.write_text(source_code_python + '\n') config = """ from universum.configuration_support import Configuration, Step @@ -322,7 +322,7 @@ def test_code_report_extended_arg_search_embedded(tmpdir: py.path.local, stdout_ ]) """ - env.configs_file.write(config) + env.configs_file.write_text(config) env.run() stdout_checker.assert_absent_calls_with_param("${CODE_REPORT_FILE}") diff --git a/tests/test_create_config.py b/tests/test_create_config.py index 618b1d57..bdd697bf 100644 --- a/tests/test_create_config.py +++ b/tests/test_create_config.py @@ -1,12 +1,12 @@ import os import subprocess -import py +import pathlib from .utils import python -def test_create_config(tmpdir: py.path.local): - launch_parameters = dict(capture_output=True, cwd=tmpdir, env=dict(os.environ, PYTHONPATH=os.getcwd())) +def test_create_config(tmp_path: pathlib.Path): + launch_parameters = dict(capture_output=True, cwd=tmp_path, env=dict(os.environ, PYTHONPATH=os.getcwd())) result = subprocess.run([python(), "-m", "universum", "init"], check=True, **launch_parameters) # type: ignore new_command = '' for line in result.stdout.splitlines(): diff --git a/tests/test_deployment.py b/tests/test_deployment.py index 86f652a2..2b4c07a0 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -9,17 +9,17 @@ def test_minimal_install(clean_docker_main: UniversumRunner): # Run locally log = clean_docker_main.run(simple_test_config, force_installed=True) - assert clean_docker_main.local.repo_file.basename in log + assert clean_docker_main.local.repo_file.name in log # Run from Git clean_docker_main.clean_artifacts() log = clean_docker_main.run(simple_test_config, vcs_type="git", force_installed=True) - assert clean_docker_main.git.repo_file.basename in log + assert clean_docker_main.git.repo_file.name in log # Run from P4 clean_docker_main.clean_artifacts() log = clean_docker_main.run(simple_test_config, vcs_type="p4", force_installed=True) - assert clean_docker_main.perforce.repo_file.basename in log + assert clean_docker_main.perforce.repo_file.name in log def test_minimal_install_with_git_only(clean_docker_main_no_p4: UniversumRunner, capsys): @@ -30,7 +30,7 @@ def test_minimal_install_with_git_only(clean_docker_main_no_p4: UniversumRunner, # Run from git clean_docker_main_no_p4.clean_artifacts() log = clean_docker_main_no_p4.run(simple_test_config, vcs_type="git", force_installed=True) - assert clean_docker_main_no_p4.git.repo_file.basename in log + assert clean_docker_main_no_p4.git.repo_file.name in log def test_minimal_install_plain_ubuntu(clean_docker_main_no_vcs: UniversumRunner, capsys): @@ -44,4 +44,4 @@ def test_minimal_install_plain_ubuntu(clean_docker_main_no_vcs: UniversumRunner, # Run locally log = clean_docker_main_no_vcs.run(simple_test_config, force_installed=True) - assert clean_docker_main_no_vcs.local.repo_file.basename in log + assert clean_docker_main_no_vcs.local.repo_file.name in log diff --git a/tests/test_git_poll.py b/tests/test_git_poll.py index 219849d1..df6f96df 100644 --- a/tests/test_git_poll.py +++ b/tests/test_git_poll.py @@ -1,7 +1,7 @@ # pylint: disable = redefined-outer-name from typing import Optional -import py +import pathlib import pytest from .conftest import FuzzyCallChecker @@ -10,8 +10,8 @@ @pytest.fixture() -def git_poll_environment(git_client: GitClient, tmpdir: py.path.local): - yield GitTestEnvironment(git_client, tmpdir, test_type="poll") +def git_poll_environment(git_client: GitClient, tmp_path: pathlib.Path): + yield GitTestEnvironment(git_client, tmp_path, test_type="poll") def make_branch_with_changes(git_server: GitServer, branch_name: str, commits_number: int, diff --git a/tests/test_github_handler.py b/tests/test_github_handler.py index f1252c32..39e6b842 100644 --- a/tests/test_github_handler.py +++ b/tests/test_github_handler.py @@ -1,7 +1,7 @@ # pylint: disable = redefined-outer-name, abstract-method +import pathlib import pytest -import py from universum.modules.vcs.github_vcs import GithubToken from .conftest import FuzzyCallChecker @@ -45,7 +45,7 @@ class GithubHandlerEnvironment(BaseTestEnvironment): }""" check_run_url: str = "http://localhost/" - def __init__(self, directory: py.path.local): + def __init__(self, directory: pathlib.Path): super().__init__(BaseVcsClient(), directory, "github-handler", "") self.settings.GithubHandler.payload = "{}" @@ -57,8 +57,8 @@ def __init__(self, directory: py.path.local): @pytest.fixture() -def github_handler_environment(tmpdir: py.path.local): - yield GithubHandlerEnvironment(tmpdir) +def github_handler_environment(tmp_path: pathlib.Path): + yield GithubHandlerEnvironment(tmp_path) @pytest.fixture(autouse=True) diff --git a/tests/test_html_output.py b/tests/test_html_output.py index d2ca0825..26031463 100644 --- a/tests/test_html_output.py +++ b/tests/test_html_output.py @@ -45,16 +45,16 @@ log_name = "universum_log" -def create_environment(test_type, tmpdir): - env = utils.LocalTestEnvironment(tmpdir, test_type) - env.configs_file.write(config) +def create_environment(test_type, tmp_path): + env = utils.LocalTestEnvironment(tmp_path, test_type) + env.configs_file.write_text(config) env.settings.Output.html_log = log_name return env @pytest.fixture(params=["main", "nonci"]) -def environment_main_and_nonci(request, tmpdir): - yield create_environment(request.param, tmpdir) +def environment_main_and_nonci(request, tmp_path): + yield create_environment(request.param, tmp_path) @pytest.fixture() @@ -66,19 +66,19 @@ def browser(): firefox.close() -def test_cli_log_custom_name(tmpdir): +def test_cli_log_custom_name(tmp_path): custom_log_name = "custom_name.html" - artifact_dir = check_cli(tmpdir, ["-hl", custom_log_name]) + artifact_dir = check_cli(tmp_path, ["-hl", custom_log_name]) assert os.path.exists(os.path.join(artifact_dir, custom_log_name)) -def test_cli_log_default_name(tmpdir): - artifact_dir = check_cli(tmpdir, ["-hl"]) +def test_cli_log_default_name(tmp_path): + artifact_dir = check_cli(tmp_path, ["-hl"]) assert os.path.exists(os.path.join(artifact_dir, "universum_log.html")) -def test_cli_no_log_requested(tmpdir): - artifact_dir = check_cli(tmpdir, []) +def test_cli_no_log_requested(tmp_path): + artifact_dir = check_cli(tmp_path, []) for file_name in os.listdir(artifact_dir): assert not file_name.endswith(".html") @@ -88,8 +88,8 @@ def test_success(environment_main_and_nonci, browser): check_html_log(environment_main_and_nonci.artifact_dir, browser) -def test_success_clean_build(tmpdir, browser): - env = create_environment("main", tmpdir) +def test_success_clean_build(tmp_path, browser): + env = create_environment("main", tmp_path) env.settings.Main.clean_build = True env.run() check_html_log(env.artifact_dir, browser) @@ -298,13 +298,13 @@ def check_timestamps(body_element, universum_log_element): assert delta.seconds <= 60 -def check_cli(tmpdir, html_log_params): - artifact_dir = tmpdir.join("artifacts") - config_file = tmpdir.join("configs.py") +def check_cli(tmp_path, html_log_params): + artifact_dir = tmp_path / "artifacts" + config_file = tmp_path / "configs.py" config_file.write_text(config, "utf-8") cli_params = ["-vt", "none", - "-fsd", str(tmpdir), + "-fsd", str(tmp_path), "-cfg", str(config_file), "-ad", str(artifact_dir)] html_log_params.extend(cli_params) diff --git a/tests/test_integration.py b/tests/test_integration.py index d7a3749a..11c567cc 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -2,9 +2,10 @@ import signal import subprocess import time +import shutil +import pathlib from typing import Any -import py import pytest from .deployment_utils import UniversumRunner, LocalSources @@ -20,7 +21,7 @@ def get_line_with_text(text: str, log: str) -> str: def test_minimal_execution(docker_main_and_nonci: UniversumRunner): log = docker_main_and_nonci.run(simple_test_config) - assert docker_main_and_nonci.local.repo_file.basename in log + assert docker_main_and_nonci.local.repo_file.name in log def test_artifacts(docker_main: UniversumRunner): @@ -223,12 +224,12 @@ def test_empty_steps(docker_main_and_nonci: UniversumRunner): def test_minimal_git(docker_main_with_vcs: UniversumRunner): log = docker_main_with_vcs.run(simple_test_config, vcs_type="git") - assert docker_main_with_vcs.git.repo_file.basename in log + assert docker_main_with_vcs.git.repo_file.name in log def test_minimal_p4(docker_main_with_vcs: UniversumRunner): log = docker_main_with_vcs.run(simple_test_config, vcs_type="p4") - assert docker_main_with_vcs.perforce.repo_file.basename in log + assert docker_main_with_vcs.perforce.repo_file.name in log def test_p4_params(docker_main_with_vcs: UniversumRunner): @@ -237,20 +238,20 @@ def test_p4_params(docker_main_with_vcs: UniversumRunner): config = f""" from universum.configuration_support import Configuration, Step -configs = Configuration([Step(name="Test step", command=["cat", "{p4_file.basename}"])]) +configs = Configuration([Step(name="Test step", command=["cat", "{p4_file.name}"])]) """ # Prepare SYNC_CHANGELIST sync_cl = p4.run_changes("-s", "submitted", "-m1", docker_main_with_vcs.perforce.depot)[0]["change"] p4.run_edit(docker_main_with_vcs.perforce.depot) - p4_file.write("This line shouldn't be in file.\n") + p4_file.write_text("This line shouldn't be in file.\n") change = p4.fetch_change() change["Description"] = "Rename basic config" p4.run_submit(change) # Prepare SHELVE_CHANGELIST p4.run_edit(docker_main_with_vcs.perforce.depot) - p4_file.write("This line should be in file.\n") + p4_file.write_text("This line should be in file.\n") change = p4.fetch_change() change["Description"] = "CL for shelving" shelve_cl = p4.save_change(change)[0].split()[1] @@ -305,8 +306,8 @@ def test_empty_required_params(docker_main_with_vcs: UniversumRunner, url_error_ def test_environment(docker_main_and_nonci: UniversumRunner): - script = docker_main_and_nonci.local.root_directory.join("script.sh") - script.write("""#!/bin/bash + script = docker_main_and_nonci.local.root_directory / "script.sh" + script.write_text("""#!/bin/bash echo ${SPECIAL_TESTING_VARIABLE} """) script.chmod(0o777) @@ -333,19 +334,19 @@ def test_environment(docker_main_and_nonci: UniversumRunner): @pytest.mark.parametrize("terminate_type", [signal.SIGINT, signal.SIGTERM], ids=["interrupt", "terminate"]) -def test_abort(local_sources: LocalSources, tmpdir: py.path.local, terminate_type): +def test_abort(local_sources: LocalSources, tmp_path: pathlib.Path, terminate_type): config = """ from universum.configuration_support import Configuration configs = Configuration([dict(name="Long step", command=["sleep", "10"])]) * 5 """ - config_file = tmpdir.join("configs.py") - config_file.write(config) + config_file = tmp_path / "configs.py" + config_file.write_text(config) with subprocess.Popen([python(), "-m", "universum", "-o", "console", "-st", "local", "-vt", "none", - "-pr", str(tmpdir.join("project_root")), - "-ad", str(tmpdir.join("artifacts")), + "-pr", str(tmp_path / "project_root"), + "-ad", str(tmp_path / "artifacts"), "-fsd", str(local_sources.root_directory), "-cfg", str(config_file)]) as process: time.sleep(5) @@ -353,29 +354,30 @@ def test_abort(local_sources: LocalSources, tmpdir: py.path.local, terminate_typ assert process.wait(5) == 3 -def test_exit_code(local_sources: LocalSources, tmpdir: py.path.local): +def test_exit_code(local_sources: LocalSources, tmp_path: pathlib.Path): config = """ from universum.configuration_support import Configuration configs = Configuration([dict(name="Unsuccessful step", command=["exit", "1"])]) """ - config_file = tmpdir.join("configs.py") - config_file.write(config) + config_file = tmp_path / "configs.py" + config_file.write_text(config) with subprocess.Popen([python(), "-m", "universum", "-o", "console", "-st", "local", "-vt", "none", - "-pr", str(tmpdir.join("project_root")), - "-ad", str(tmpdir.join("artifacts")), + "-pr", str(tmp_path / "project_root"), + "-ad", str(tmp_path / "artifacts"), "-fsd", str(local_sources.root_directory), "-cfg", str(config_file)]) as process: assert process.wait() == 0 - tmpdir.join("artifacts").remove(rec=True) + artifacts_dir = tmp_path / "artifacts" + shutil.rmtree(str(artifacts_dir)) with subprocess.Popen([python(), "-m", "universum", "--fail-unsuccessful", "-o", "console", "-st", "local", "-vt", "none", - "-pr", str(tmpdir.join("project_root")), - "-ad", str(tmpdir.join("artifacts")), + "-pr", str(tmp_path / "project_root"), + "-ad", str(tmp_path / "artifacts"), "-fsd", str(local_sources.root_directory), "-cfg", str(config_file)]) as process: assert process.wait() == 1 diff --git a/tests/test_nonci.py b/tests/test_nonci.py index 7d33eb61..fe24168d 100644 --- a/tests/test_nonci.py +++ b/tests/test_nonci.py @@ -23,7 +23,7 @@ def test_launcher_output(docker_nonci: UniversumRunner): - version control and review system are not used - project root is set to current directory """ - cwd = docker_nonci.local.root_directory.strpath + cwd = str(docker_nonci.local.root_directory) artifacts = docker_nonci.artifact_dir file_output_expected = f"Adding file {artifacts}/test_step_log.txt to artifacts" pwd_string_in_logs = f"pwd:[{cwd}]" diff --git a/tests/test_p4_exception_handling.py b/tests/test_p4_exception_handling.py index 5b510fa7..d88bd2ff 100644 --- a/tests/test_p4_exception_handling.py +++ b/tests/test_p4_exception_handling.py @@ -1,6 +1,8 @@ # pylint: disable = redefined-outer-name -import py +import os +import shutil +import pathlib import pytest from universum import __main__ @@ -9,8 +11,8 @@ @pytest.fixture() -def perforce_environment(perforce_workspace: PerforceWorkspace, tmpdir: py.path.local): - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="main") +def perforce_environment(perforce_workspace: PerforceWorkspace, tmp_path: pathlib.Path): + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="main") def test_p4_forbidden_local_revert(perforce_environment: P4TestEnvironment, stdout_checker: FuzzyCallChecker): @@ -26,8 +28,8 @@ def test_p4_forbidden_local_revert(perforce_environment: P4TestEnvironment, stdo perforce_environment.shelve_config(config) result = __main__.run(perforce_environment.settings) # Clean up the directory at once to make sure it doesn't remain non-writable even if some assert fails - perforce_environment.temp_dir.chmod(0o0777, rec=1) - perforce_environment.temp_dir.remove(rec=1) + os.system(f"chmod -R 777 {perforce_environment.temp_dir}") + shutil.rmtree(str(perforce_environment.temp_dir)) assert result == 0 diff --git a/tests/test_p4_revert_unshelved.py b/tests/test_p4_revert_unshelved.py index a2253932..997876a9 100644 --- a/tests/test_p4_revert_unshelved.py +++ b/tests/test_p4_revert_unshelved.py @@ -25,7 +25,7 @@ def __init__(self, perforce_workspace: PerforceWorkspace): settings.PerforceVcs.user = perforce_workspace.server.user settings.PerforceVcs.password = perforce_workspace.server.password settings.Output.type = "term" - settings.ProjectDirectory.project_root = str(self.perforce_workspace.root_directory + "/../new_workspace") + settings.ProjectDirectory.project_root = str(self.perforce_workspace.root_directory) + "/../new_workspace" settings.PerforceWithMappings.mappings = [self.perforce_workspace.depot + " /..."] settings.PerforceMainVcs.force_clean = True settings.PerforceMainVcs.client = "new_client" @@ -61,13 +61,14 @@ def diff_parameters(perforce_workspace: PerforceWorkspace): def test_p4_c_and_revert(diff_parameters): # pylint: disable = too-many-locals p4 = diff_parameters.perforce_workspace.p4 - test_dir = diff_parameters.perforce_workspace.root_directory.ensure("test_files", dir=True) + test_dir = diff_parameters.perforce_workspace.root_directory / "test_files" + test_dir.mkdir() def create_file(filename): - cur_file = test_dir.join(filename) + cur_file = test_dir / filename p4.run("add", str(cur_file)) p4.run("edit", str(cur_file)) - cur_file.write(f"import os\n\nprint('File {0} has no special modifiers.')\n" + cur_file.write_text(f"import os\n\nprint('File {0} has no special modifiers.')\n" f"print(os.name)\nprint(os.getcwd())\nprint(os.strerror(3))\n" f"print('File change type: {filename}')\n") @@ -84,27 +85,27 @@ def create_file(filename): p4.run_submit(change) # open for edit for edit, move/add and move/rename files - for cur_file in [test_dir.join("edit"), test_dir.join("move"), test_dir.join("rename")]: + for cur_file in [test_dir / "edit", test_dir / "move", test_dir / "rename"]: p4.run("edit", str(cur_file)) # edit - test_dir.join("edit").write(f"import os\n\nprint('File {test_dir.join('edit')} has no special modifiers.')\n" + (test_dir / "edit").write_text(f"import os\n\nprint('File {test_dir / 'edit'} has no special modifiers.')\n" f"print(os.name)\nprint(os.getegid())\nprint(os.ctermid())\n" f"print('File change type: \"edit\"')\n") # move, rename - p4.run("move", test_dir + "/move", test_dir + "/moved/move") - p4.run("rename", test_dir + "/rename", test_dir + "/renamed_rename") + p4.run("move", test_dir / "move", test_dir / "moved/move") + p4.run("rename", test_dir / "rename", test_dir / "renamed_rename") # integrate - p4.run("integrate", test_dir + "/integrate", test_dir + "/integrated") + p4.run("integrate", test_dir / "integrate", test_dir / "integrated") p4.run("resolve", "-at") # branch - p4.run("integrate", test_dir + "/branch", test_dir + "/moved/branch_to") + p4.run("integrate", test_dir / "branch", test_dir / "moved/branch_to") # delete - p4.run("delete", test_dir + "/delete") + p4.run("delete", test_dir / "delete") # add - add = test_dir.join("add") - add.write("\nprint('new file')\nprint('not in repo, only for shelve.')") + add = test_dir / "add" + add.write_text("\nprint('new file')\nprint('not in repo, only for shelve.')") p4.run("add", add) # make change diff --git a/tests/test_p4_submit.py b/tests/test_p4_submit.py index 83675ac2..dee0bba6 100644 --- a/tests/test_p4_submit.py +++ b/tests/test_p4_submit.py @@ -1,6 +1,6 @@ # pylint: disable = redefined-outer-name -import py +import pathlib import pytest from . import utils @@ -8,8 +8,8 @@ @pytest.fixture() -def p4_submit_environment(perforce_workspace: PerforceWorkspace, tmpdir: py.path.local): - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="submit") +def p4_submit_environment(perforce_workspace: PerforceWorkspace, tmp_path: pathlib.Path): + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="submit") def test_fail_changing_non_checked_out_file(p4_submit_environment: P4TestEnvironment): @@ -28,7 +28,7 @@ def test_success_changing_checked_out_file(p4_submit_environment: P4TestEnvironm p4_submit_environment.vcs_client.p4.run("edit", str(target_file)) text = utils.randomize_name("This is change ") - target_file.write(text + "\n") + target_file.write_text(text + "\n") change = p4_submit_environment.vcs_client.p4.run_change("-o")[0] change["Description"] = "Test submit" diff --git a/tests/test_poll.py b/tests/test_poll.py index 7f1a53c7..1b6d4d44 100644 --- a/tests/test_poll.py +++ b/tests/test_poll.py @@ -1,7 +1,7 @@ # pylint: disable = redefined-outer-name from typing import Union -import py +import pathlib import pytest from universum import __main__ @@ -11,15 +11,15 @@ from .utils import LocalTestEnvironment -def test_poll_local_vcs(tmpdir: py.path.local): - env = LocalTestEnvironment(tmpdir, "poll") +def test_poll_local_vcs(tmp_path: pathlib.Path): + env = LocalTestEnvironment(tmp_path, "poll") env.run() def test_p4_success_command_line_no_changes(stdout_checker: FuzzyCallChecker, perforce_workspace: PerforceWorkspace, - tmpdir: py.path.local): - db_file = tmpdir.join("p4poll.json") + tmp_path: pathlib.Path): + db_file = tmp_path / "p4poll.json" result = __main__.main(["poll", "-ot", "term", "-vt", "p4", "-f", str(db_file), @@ -34,8 +34,8 @@ def test_p4_success_command_line_no_changes(stdout_checker: FuzzyCallChecker, def test_git_success_command_line_no_changes(stdout_checker: FuzzyCallChecker, git_server: GitServer, - tmpdir: py.path.local): - db_file = tmpdir.join("gitpoll.json") + tmp_path: pathlib.Path): + db_file = tmp_path / "gitpoll.json" result = __main__.main(["poll", "-ot", "term", "-vt", "git", "-f", str(db_file), @@ -48,8 +48,8 @@ def test_git_success_command_line_no_changes(stdout_checker: FuzzyCallChecker, def test_p4_error_command_line_wrong_port(stdout_checker: FuzzyCallChecker, perforce_workspace: PerforceWorkspace, - tmpdir: py.path.local): - db_file = tmpdir.join("p4poll.json") + tmp_path: pathlib.Path): + db_file = tmp_path / "p4poll.json" result = __main__.main(["poll", "-ot", "term", "-vt", "p4", "-f", str(db_file), @@ -64,8 +64,8 @@ def test_p4_error_command_line_wrong_port(stdout_checker: FuzzyCallChecker, def test_git_error_command_line_wrong_port(stdout_checker: FuzzyCallChecker, git_server: GitServer, - tmpdir: py.path.local): - db_file = tmpdir.join("gitpoll.json") + tmp_path: pathlib.Path): + db_file = tmp_path / "gitpoll.json" result = __main__.main(["poll", "-ot", "term", "-vt", "git", "-f", str(db_file), @@ -77,11 +77,11 @@ def test_git_error_command_line_wrong_port(stdout_checker: FuzzyCallChecker, @pytest.fixture(params=["git", "p4"]) -def poll_environment(request, perforce_workspace: PerforceWorkspace, git_client: GitClient, tmpdir: py.path.local): +def poll_environment(request, perforce_workspace: PerforceWorkspace, git_client: GitClient, tmp_path: pathlib.Path): if request.param == "git": - yield GitTestEnvironment(git_client, tmpdir, test_type="poll") + yield GitTestEnvironment(git_client, tmp_path, test_type="poll") else: - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="poll") + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="poll") def test_error_one_change(stdout_checker: FuzzyCallChecker, log_exception_checker: FuzzyCallChecker, diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index fe597c7c..6d067569 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -13,10 +13,10 @@ class ArtifactsTestEnvironment(LocalTestEnvironment): - def __init__(self, tmpdir: pathlib.Path) -> None: - super().__init__(tmpdir, "main") + def __init__(self, tmp_path: pathlib.Path) -> None: + super().__init__(tmp_path, "main") self.artifact_name: str = "artifact" - self.artifact_path: pathlib.Path = self.artifact_dir.join(self.artifact_name) + self.artifact_path: pathlib.Path = self.artifact_dir / self.artifact_name self.artifact_content: str = "artifact content" def write_config_file(self, artifact_prebuild_clean: bool) -> None: @@ -40,14 +40,14 @@ def check_step_artifact_absent(self) -> None: assert not os.path.exists(self.artifact_path) def create_artifact_file(self) -> None: - precreated_artifact: pathlib.Path = self.src_dir.join(self.artifact_name) + precreated_artifact: pathlib.Path = self.src_dir / self.artifact_name with open(precreated_artifact, "w", encoding="utf-8") as f: f.write("pre-created artifact content") @pytest.fixture() -def test_env(tmpdir: pathlib.Path) -> Generator[ArtifactsTestEnvironment, None, None]: - yield ArtifactsTestEnvironment(tmpdir) +def test_env(tmp_path: pathlib.Path) -> Generator[ArtifactsTestEnvironment, None, None]: + yield ArtifactsTestEnvironment(tmp_path) def test_no_artifact_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: diff --git a/tests/test_regression.py b/tests/test_regression.py index 2740f585..cc9223f4 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -1,6 +1,6 @@ # pylint: disable = redefined-outer-name -import py +import pathlib import pytest import P4 @@ -13,7 +13,7 @@ def test_which_universum_is_tested(docker_main: UniversumRunner, pytestconfig): # THIS TEST PATCHES ACTUAL SOURCES! Discretion is advised - init_file = pytestconfig.rootpath.joinpath("universum", "__init__.py") + init_file = pytestconfig.rootpath / "universum" / "__init__.py" backup = init_file.read_bytes() test_line = utils.randomize_name("THIS IS A TESTING VERSION") init_file.write_text(f"""__title__ = "Universum" @@ -42,14 +42,14 @@ def test_teardown_fixture_output_verification(print_text_on_teardown: None): @pytest.mark.parametrize("should_not_execute", [True, False], ids=['no-sources', 'deleted-sources']) -def test_clean_sources_exception(tmpdir: py.path.local, stdout_checker: FuzzyCallChecker, should_not_execute): - env = LocalTestEnvironment(tmpdir, "main") +def test_clean_sources_exception(tmp_path: pathlib.Path, stdout_checker: FuzzyCallChecker, should_not_execute): + env = LocalTestEnvironment(tmp_path, "main") env.settings.Vcs.type = "none" - source_directory = tmpdir + source_directory = tmp_path if should_not_execute: source_directory = source_directory / 'nonexisting_dir' env.settings.LocalMainVcs.source_dir = str(source_directory) - env.configs_file.write(f""" + env.configs_file.write_text(f""" from universum.configuration_support import Configuration configs = Configuration([dict(name="Test configuration", @@ -84,8 +84,8 @@ def test_non_utf8_environment(docker_main: UniversumRunner): @pytest.fixture() -def perforce_environment(perforce_workspace: PerforceWorkspace, tmpdir: py.path.local): - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="main") +def perforce_environment(perforce_workspace: PerforceWorkspace, tmp_path: pathlib.Path): + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="main") def test_p4_multiple_spaces_in_mappings(perforce_environment: P4TestEnvironment): @@ -102,7 +102,7 @@ def test_p4_repository_difference_format(perforce_environment: P4TestEnvironment """ perforce_environment.shelve_config(config) perforce_environment.run() - diff = perforce_environment.artifact_dir.join('REPOSITORY_DIFFERENCE.txt').read() + diff = (perforce_environment.artifact_dir / 'REPOSITORY_DIFFERENCE.txt').read_text() assert "This is a changed step name" in diff assert "b'" not in diff @@ -134,7 +134,7 @@ def test_p4_api_failed_opened(perforce_environment: P4TestEnvironment, mock_open perforce_environment.settings.Launcher.output = "file" perforce_environment.run() - log = perforce_environment.artifact_dir.join(f'{step_name}_log.txt').read() + log = (perforce_environment.artifact_dir / f'{step_name}_log.txt').read_text() assert "Module sh got exit code 1" in log assert "Getting file diff failed due to Perforce server internal error" in log @@ -168,7 +168,7 @@ def perforce_environment_with_files(perforce_environment: P4TestEnvironment): yield {"env": perforce_environment, "files": files} for entry in files: - perforce_environment.vcs_client.delete_file(entry.basename) + perforce_environment.vcs_client.delete_file(entry.name) def test_success_p4_resolve_unshelved(perforce_environment_with_files: dict, stdout_checker: FuzzyCallChecker): @@ -177,7 +177,7 @@ def test_success_p4_resolve_unshelved(perforce_environment_with_files: dict, std config = f""" from universum.configuration_support import Step, Configuration -configs = Configuration([Step(name="Print file", command=["bash", "-c", "cat '{p4_file.basename}'"])]) +configs = Configuration([Step(name="Print file", command=["bash", "-c", "cat '{p4_file.name}'"])]) """ env.shelve_config(config) cls = [env.vcs_client.shelve_file(p4_file, "This is changed line 1\nThis is unchanged line 2"), @@ -195,7 +195,7 @@ def test_fail_p4_resolve_unshelved(perforce_environment_with_files: dict, stdout config = f""" from universum.configuration_support import Step, Configuration -configs = Configuration([Step(name="Print file", command=["bash", "-c", "cat '{p4_file.basename}'"])]) +configs = Configuration([Step(name="Print file", command=["bash", "-c", "cat '{p4_file.name}'"])]) """ env.shelve_config(config) cls = [env.vcs_client.shelve_file(p4_file, "This is changed line 1\nThis is unchanged line 2"), @@ -204,7 +204,7 @@ def test_fail_p4_resolve_unshelved(perforce_environment_with_files: dict, stdout env.run(expect_failure=True) stdout_checker.assert_has_calls_with_param("Problems during merge while resolving shelved CLs") - stdout_checker.assert_has_calls_with_param(str(p4_file.basename)) + stdout_checker.assert_has_calls_with_param(str(p4_file.name)) def test_success_p4_resolve_unshelved_multiple(perforce_environment_with_files: dict): @@ -223,6 +223,6 @@ def test_success_p4_resolve_unshelved_multiple(perforce_environment_with_files: env.settings.PerforceMainVcs.shelve_cls.extend([cl_1, cl_2]) env.run() - repo_state = env.artifact_dir.join('REPOSITORY_STATE.txt').read() - assert p4_files[0].basename in repo_state - assert p4_files[1].basename in repo_state + repo_state = (env.artifact_dir / 'REPOSITORY_STATE.txt').read_text() + assert p4_files[0].name in repo_state + assert p4_files[1].name in repo_state diff --git a/tests/test_report.py b/tests/test_report.py index 6ae9dab6..3b3da310 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,7 +1,7 @@ # pylint: disable = redefined-outer-name, abstract-method +import pathlib import pytest -import py from universum.modules.vcs.github_vcs import GithubToken from . import utils @@ -9,7 +9,7 @@ class ReportEnvironment(utils.BaseTestEnvironment): - def __init__(self, client: GitClient, directory: py.path.local): + def __init__(self, client: GitClient, directory: pathlib.Path): super().__init__(client, directory, "main", "") self.settings.Vcs.type = "github" @@ -30,8 +30,8 @@ def __init__(self, client: GitClient, directory: py.path.local): @pytest.fixture() -def report_environment(git_client: GitClient, tmpdir: py.path.local): - yield ReportEnvironment(git_client, tmpdir) +def report_environment(git_client: GitClient, tmp_path: pathlib.Path): + yield ReportEnvironment(git_client, tmp_path) def test_github_run(report_environment: ReportEnvironment, monkeypatch): diff --git a/tests/test_run_steps_filter.py b/tests/test_run_steps_filter.py index 4eadb979..86232efd 100644 --- a/tests/test_run_steps_filter.py +++ b/tests/test_run_steps_filter.py @@ -40,12 +40,12 @@ def step(name, cmd=False): @pytest.mark.parametrize("test_type", test_types) @pytest.mark.parametrize("filters, expected_logs, unexpected_logs", filters_parametrize_values) -def test_steps_filter(tmpdir, stdout_checker, filters, expected_logs, unexpected_logs, test_type): - params = get_cli_params(test_type, tmpdir) +def test_steps_filter(tmp_path, stdout_checker, filters, expected_logs, unexpected_logs, test_type): + params = get_cli_params(test_type, tmp_path) params.extend(["-o", "console"]) for _filter in filters: params.append(f"-f={_filter}") - params.extend(["-cfg", get_config_file_path(tmpdir, config)]) + params.extend(["-cfg", get_config_file_path(tmp_path, config)]) return_code = __main__.main(params) @@ -57,39 +57,39 @@ def test_steps_filter(tmpdir, stdout_checker, filters, expected_logs, unexpected @pytest.mark.parametrize("test_type", test_types) -def test_steps_filter_no_match(tmpdir, stdout_checker, test_type): +def test_steps_filter_no_match(tmp_path, stdout_checker, test_type): include_pattern = "asdf" exclude_pattern = "qwer" - cli_params = get_cli_params(test_type, tmpdir) + cli_params = get_cli_params(test_type, tmp_path) cli_params.extend(["-f", f"{include_pattern}:!{exclude_pattern}"]) - check_empty_config_error(tmpdir, stdout_checker, cli_params) + check_empty_config_error(tmp_path, stdout_checker, cli_params) stdout_checker.assert_has_calls_with_param(include_pattern) stdout_checker.assert_has_calls_with_param(exclude_pattern) @pytest.mark.parametrize("test_type", test_types) -def test_config_empty(tmpdir, stdout_checker, test_type): - check_empty_config_error(tmpdir, stdout_checker, get_cli_params(test_type, tmpdir)) +def test_config_empty(tmp_path, stdout_checker, test_type): + check_empty_config_error(tmp_path, stdout_checker, get_cli_params(test_type, tmp_path)) -def check_empty_config_error(tmpdir, stdout_checker, cli_params): - cli_params.extend(["-cfg", get_config_file_path(tmpdir, empty_config)]) +def check_empty_config_error(tmp_path, stdout_checker, cli_params): + cli_params.extend(["-cfg", get_config_file_path(tmp_path, empty_config)]) return_code = __main__.main(cli_params) assert return_code == 1 stdout_checker.assert_has_calls_with_param("Project configs are empty") -def get_config_file_path(tmpdir, text): - config_file = tmpdir.join("configs.py") +def get_config_file_path(tmp_path, text): + config_file = tmp_path / "configs.py" config_file.write_text(text, "utf-8") return str(config_file) -def get_cli_params(test_type, tmpdir): +def get_cli_params(test_type, tmp_path): if test_type == "nonci": return ["nonci"] return ["-vt", "none", - "-fsd", str(tmpdir), + "-fsd", str(tmp_path), "--clean-build"] diff --git a/tests/test_submit.py b/tests/test_submit.py index d6532ce6..432f8d10 100644 --- a/tests/test_submit.py +++ b/tests/test_submit.py @@ -3,7 +3,7 @@ import os from typing import Callable, Union import shutil -import py +import pathlib import pytest from . import utils @@ -13,16 +13,18 @@ @pytest.fixture() -def p4_submit_environment(perforce_workspace: PerforceWorkspace, tmpdir: py.path.local): - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="submit") +def p4_submit_environment(perforce_workspace: PerforceWorkspace, tmp_path: pathlib.Path): + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="submit") @pytest.mark.parametrize("branch", ["write-protected", "trigger-protected"]) def test_p4_error_forbidden_branch(p4_submit_environment: P4TestEnvironment, branch: str): - protected_dir = p4_submit_environment.vcs_client.root_directory.mkdir(branch) - file_to_add = protected_dir.join(utils.randomize_name("new_file") + ".txt") + protected_dir = p4_submit_environment.vcs_client.root_directory / branch + protected_dir.mkdir() + file_name = utils.randomize_name("new_file") + ".txt" + file_to_add = protected_dir / file_name text = "This is a new line in the file" - file_to_add.write(text + "\n") + file_to_add.write_text(text + "\n") p4_submit_environment.settings.Submit.reconcile_list = str(file_to_add) @@ -41,17 +43,17 @@ def test_p4_success_files_in_default(p4_submit_environment: P4TestEnvironment): p4_file = p4_submit_environment.vcs_client.repo_file p4.run_edit(str(p4_file)) text = "This text should be in file" - p4_file.write(text + "\n") + p4_file.write_text(text + "\n") # This file should be successfully submitted file_name = utils.randomize_name("new_file") + ".txt" - new_file = p4_submit_environment.vcs_client.root_directory.join(file_name) - new_file.write("This is a new file" + "\n") + new_file = p4_submit_environment.vcs_client.root_directory / file_name + new_file.write_text("This is a new file" + "\n") p4_submit_environment.settings.Submit.reconcile_list = str(new_file) p4_submit_environment.run() - assert text in p4_file.read() + assert text in p4_file.read_text() def test_p4_error_files_in_default_and_reverted(p4_submit_environment: P4TestEnvironment): @@ -60,19 +62,21 @@ def test_p4_error_files_in_default_and_reverted(p4_submit_environment: P4TestEnv p4_file = p4_submit_environment.vcs_client.repo_file p4.run_edit(str(p4_file)) text_default = "This text should be in file" - p4_file.write(text_default + "\n") + p4_file.write_text(text_default + "\n") # This file must fail submit and remain unchanged while not checked out any more - protected_dir = p4_submit_environment.vcs_client.root_directory.mkdir("write-protected") - new_file = protected_dir.join(utils.randomize_name("new_file") + ".txt") + protected_dir = p4_submit_environment.vcs_client.root_directory / "write-protected" + protected_dir.mkdir() + file_name = utils.randomize_name("new_file") + ".txt" + new_file = protected_dir / file_name text_new = "This is a new line in the file" - new_file.write(text_new + "\n") + new_file.write_text(text_new + "\n") p4_submit_environment.settings.Submit.reconcile_list = str(new_file) p4_submit_environment.run(expect_failure=True) - assert text_default in p4_file.read() - assert text_new in new_file.read() + assert text_default in p4_file.read_text() + assert text_new in new_file.read_text() class SubmitterParameters: @@ -110,11 +114,11 @@ def inner(environment): @pytest.fixture(params=["git", "p4"]) -def submit_environment(request, perforce_workspace: PerforceWorkspace, git_client: GitClient, tmpdir: py.path.local): +def submit_environment(request, perforce_workspace: PerforceWorkspace, git_client: GitClient, tmp_path: pathlib.Path): if request.param == "git": - yield GitTestEnvironment(git_client, tmpdir, test_type="submit") + yield GitTestEnvironment(git_client, tmp_path, test_type="submit") else: - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="submit") + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="submit") def test_error_no_repo(submit_environment: Union[GitTestEnvironment, P4TestEnvironment], stdout_checker: FuzzyCallChecker): @@ -138,22 +142,22 @@ def test_success_commit_add_modify_remove_one_file(submit_parameters: Callable, parameters = submit_parameters(submit_environment) file_name = utils.randomize_name("new_file") + ".txt" - temp_file = parameters.environment.vcs_client.root_directory.join(file_name) + temp_file = parameters.environment.vcs_client.root_directory / file_name file_path = str(temp_file) # Add a file - temp_file.write("This is a new file" + "\n") + temp_file.write_text("This is a new file" + "\n") parameters.assert_submit_success([file_path]) assert parameters.file_present(file_path) # Modify a file text = "This is a new line in the file" - temp_file.write(text + "\n") + temp_file.write_text(text + "\n") parameters.assert_submit_success([file_path]) assert parameters.text_in_file(text, file_path) # Delete a file - temp_file.remove() + temp_file.unlink() parameters.assert_submit_success([file_path]) assert not parameters.file_present(file_path) @@ -163,8 +167,8 @@ def test_success_ignore_new_and_deleted_while_edit_only(submit_parameters: Calla parameters = submit_parameters(submit_environment) new_file_name = utils.randomize_name("new_file") + ".txt" - temp_file = parameters.environment.vcs_client.root_directory.join(new_file_name) - temp_file.write("This is a new temp file" + "\n") + temp_file = parameters.environment.vcs_client.root_directory / new_file_name + temp_file.write_text("This is a new temp file" + "\n") deleted_file_path = str(parameters.environment.vcs_client.repo_file) deleted_file_name = os.path.basename(deleted_file_path) os.remove(deleted_file_path) @@ -184,7 +188,7 @@ def test_success_commit_modified_while_edit_only(submit_parameters: Callable, target_file = parameters.environment.vcs_client.repo_file text = utils.randomize_name("This is change ") - target_file.write(text + "\n") + target_file.write_text(text + "\n") parameters.assert_submit_success([str(target_file)], edit_only=True) assert parameters.text_in_file(text, str(target_file)) @@ -194,7 +198,7 @@ def test_error_review(submit_parameters: Callable, submit_environment: Union[Git parameters = submit_parameters(submit_environment) target_file = parameters.environment.vcs_client.repo_file - target_file.write("This is some change") + target_file.write_text("This is some change") parameters.submit_path_list([str(target_file)], review=True, expect_failure=True) parameters.stdout_checker.assert_has_calls_with_param("not supported") @@ -207,21 +211,23 @@ def test_success_reconcile_directory(submit_parameters: Callable, dir_name = utils.randomize_name("new_directory") # Create and reconcile new directory - tmp_dir = parameters.environment.vcs_client.root_directory.mkdir(dir_name) + tmp_dir = parameters.environment.vcs_client.root_directory / dir_name + tmp_dir.mkdir() for i in range(0, 9): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write("This is some file" + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text("This is some file" + "\n") parameters.assert_submit_success([str(tmp_dir) + "/"]) for i in range(0, 9): - file_path = tmp_dir.join(f"new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert parameters.file_present(str(file_path)) # Create and reconcile a directory in a directory - another_dir = tmp_dir.mkdir("another_directory") - tmp_file = another_dir.join("new_file.txt") - tmp_file.write("This is some file" + "\n") + another_dir = tmp_dir / "another_directory" + another_dir.mkdir() + tmp_file = another_dir / "new_file.txt" + tmp_file.write_text("This is some file" + "\n") parameters.assert_submit_success([str(tmp_dir) + "/"]) assert parameters.file_present(str(tmp_file)) @@ -229,13 +235,13 @@ def test_success_reconcile_directory(submit_parameters: Callable, # Modify some vcs text = utils.randomize_name("This is change ") for i in range(0, 9, 2): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") parameters.assert_submit_success([str(tmp_dir) + "/"], edit_only=True) for i in range(0, 9, 2): - file_path = tmp_dir.join(f"/new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert parameters.text_in_file(text, str(file_path)) parameters.environment.settings.Submit.edit_only = False @@ -252,106 +258,109 @@ def test_success_reconcile_wildcard(submit_parameters: Callable, dir_name = utils.randomize_name("new_directory") # Create embedded directories, partially reconcile - tmp_dir = parameters.environment.vcs_client.root_directory.mkdir(dir_name) - inner_dir = tmp_dir.mkdir("inner_directory") + tmp_dir = parameters.environment.vcs_client.root_directory / dir_name + tmp_dir.mkdir() + inner_dir = tmp_dir / "inner_directory" + inner_dir.mkdir() text = "This is some file" + "\n" for i in range(0, 9): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write(text) - tmp_file = tmp_dir.join(f"another_file{i}.txt") - tmp_file.write(text) - tmp_file = inner_dir.join(f"new_file{i}.txt") - tmp_file.write(text) + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text(text) + tmp_file = tmp_dir / f"another_file{i}.txt" + tmp_file.write_text(text) + tmp_file = inner_dir / f"new_file{i}.txt" + tmp_file.write_text(text) parameters.assert_submit_success([str(tmp_dir) + "/new_file*.txt"]) for i in range(0, 9): file_name = f"new_file{i}.txt" - file_path = tmp_dir.join(file_name) + file_path = tmp_dir / file_name assert parameters.file_present(str(file_path)) - file_path = inner_dir.join(file_name) + file_path = inner_dir / file_name assert not parameters.file_present(str(file_path)) file_name = f"another_file{i}.txt" - file_path = tmp_dir.join(file_name) + file_path = tmp_dir / file_name assert not parameters.file_present(str(file_path)) # Create one more directory other_dir_name = utils.randomize_name("new_directory") - other_tmp_dir = parameters.environment.vcs_client.root_directory.mkdir(other_dir_name) + other_tmp_dir = parameters.environment.vcs_client.root_directory / other_dir_name + other_tmp_dir.mkdir() for i in range(0, 9): - tmp_file = other_tmp_dir.join(f"new_file{i}.txt") - tmp_file.write("This is some file" + "\n") + tmp_file = other_tmp_dir / f"new_file{i}.txt" + tmp_file.write_text("This is some file" + "\n") parameters.assert_submit_success([str(parameters.environment.vcs_client.root_directory) + "/new_directory*/"]) for i in range(0, 9): file_name = f"new_file{i}.txt" - file_path = other_tmp_dir.join(file_name) + file_path = other_tmp_dir / file_name assert parameters.file_present(str(file_path)) - file_path = inner_dir.join(file_name) + file_path = inner_dir / file_name assert parameters.file_present(str(file_path)) file_name = f"another_file{i}.txt" - file_path = tmp_dir.join(file_name) + file_path = tmp_dir / file_name assert parameters.file_present(str(file_path)) # Modify some vcs text = utils.randomize_name("This is change ") for i in range(0, 9, 2): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = inner_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = tmp_dir.join(f"another_file{i}.txt") - tmp_file.write(text + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = inner_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = tmp_dir / f"another_file{i}.txt" + tmp_file.write_text(text + "\n") parameters.assert_submit_success([str(tmp_dir) + "/new_file*.txt"], edit_only=True) for i in range(0, 9, 2): - file_path = tmp_dir.join(f"/new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert parameters.text_in_file(text, str(file_path)) - file_path = inner_dir.join(f"/new_file{i}.txt") + file_path = inner_dir / f"new_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) - file_path = tmp_dir.join(f"/another_file{i}.txt") + file_path = tmp_dir / f"another_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) # Test subdirectory wildcard text = utils.randomize_name("This is change ") for i in range(1, 9, 2): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = inner_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = tmp_dir.join(f"another_file{i}.txt") - tmp_file.write(text + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = inner_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = tmp_dir / f"another_file{i}.txt" + tmp_file.write_text(text + "\n") parameters.assert_submit_success([str(tmp_dir) + "/*/*.txt"]) for i in range(1, 9, 2): - file_path = inner_dir.join(f"new_file{i}.txt") + file_path = inner_dir / f"new_file{i}.txt" assert parameters.text_in_file(text, str(file_path)) - file_path = tmp_dir.join(f"new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) - file_path = tmp_dir.join(f"another_file{i}.txt") + file_path = tmp_dir / f"another_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) # Test edit-only subdirectory wildcard text = utils.randomize_name("This is change ") for i in range(0, 9, 3): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = inner_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = tmp_dir.join("another_file{i}.txt") - tmp_file.write(text + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = inner_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = tmp_dir / "another_file{i}.txt" + tmp_file.write_text(text + "\n") parameters.assert_submit_success([str(tmp_dir) + "/*/*.txt"], edit_only=True) for i in range(0, 9, 3): - file_path = inner_dir.join(f"new_file{i}.txt") + file_path = inner_dir / f"new_file{i}.txt" assert parameters.text_in_file(text, str(file_path)) - file_path = tmp_dir.join(f"new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) - file_path = tmp_dir.join(f"another_file{i}.txt") + file_path = tmp_dir / f"another_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) parameters.environment.settings.Submit.edit_only = False @@ -370,21 +379,22 @@ def test_success_reconcile_partial(submit_parameters: Callable, parameters = submit_parameters(submit_environment) dir_name = utils.randomize_name("new_directory") - tmp_dir = parameters.environment.vcs_client.root_directory.mkdir(dir_name) + tmp_dir = parameters.environment.vcs_client.root_directory / dir_name + tmp_dir.mkdir() for i in range(0, 9): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write("This is some file" + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text("This is some file" + "\n") - reconcile_list = [str(tmp_dir.join(f"new_file{i}.txt")) for i in range(0, 4)] + reconcile_list = [str(tmp_dir / f"new_file{i}.txt") for i in range(0, 4)] reconcile_list.extend(["", " ", "\n"]) parameters.assert_submit_success(reconcile_list) for i in range(0, 4): - file_path = tmp_dir.join(f"new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert parameters.file_present(str(file_path)) for i in range(5, 9): - file_path = tmp_dir.join(f"new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert not parameters.file_present(str(file_path)) # Delete a directory diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 1bb00343..f047ea38 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,7 +1,7 @@ # pylint: disable = redefined-outer-name +import pathlib import git -import py import pytest from . import utils @@ -10,24 +10,28 @@ @pytest.fixture -def unicode_dir(tmpdir: py.path.local): - yield tmpdir.mkdir("Юніко́д з пробелами") +def unicode_dir(tmp_path: pathlib.Path): + unicode_dir_path = tmp_path / "Юніко́д з пробелами" + unicode_dir_path.mkdir() + yield unicode_dir_path @pytest.mark.parametrize("vcs", ["git", "p4"]) @pytest.mark.parametrize("test_type", ["main", "poll", "submit"]) -def test_unicode(vcs, test_type, perforce_workspace: PerforceWorkspace, git_client: GitClient, unicode_dir: py.path.local): +def test_unicode(vcs, test_type, perforce_workspace: PerforceWorkspace, git_client: GitClient, unicode_dir: pathlib.Path): env: utils.BaseTestEnvironment if vcs == "git": # change git client root dir to unicode path - work_dir = unicode_dir.mkdir("client") + work_dir = unicode_dir / "client" + work_dir.mkdir() git_client.repo = git.Repo.clone_from(git_client.server.url, work_dir) git_client.root_directory = work_dir env = GitTestEnvironment(git_client, unicode_dir, test_type=test_type) elif vcs == "p4": # change workspace root dir to unicode path - root = unicode_dir.mkdir("workspace") + root = unicode_dir / "workspace" + root.mkdir() client = perforce_workspace.p4.fetch_client(perforce_workspace.client_name) client["Root"] = str(root) perforce_workspace.root_directory = root @@ -39,16 +43,18 @@ def test_unicode(vcs, test_type, perforce_workspace: PerforceWorkspace, git_clie assert False, "Unsupported vcs type" if test_type == "submit": - temp_file = env.vcs_client.root_directory.join(utils.randomize_name("new_file") + ".txt") - temp_file.write("This is a new file" + "\n") + file_name = utils.randomize_name("new_file") + ".txt" + temp_file = env.vcs_client.root_directory / file_name + temp_file.write_text("This is a new file" + "\n") env.settings.Submit.reconcile_list = str(temp_file) env.run() -def test_unicode_main_local_vcs(unicode_dir: py.path.local): - work_dir = unicode_dir.mkdir("local_sources") - work_dir.join("source_file").write("Source file contents") +def test_unicode_main_local_vcs(unicode_dir: pathlib.Path): + work_dir = unicode_dir / "local_sources" + work_dir.mkdir() + (work_dir / "source_file").write_text("Source file contents") env = utils.LocalTestEnvironment(unicode_dir, "main") env.settings.Vcs.type = "none" diff --git a/tests/utils.py b/tests/utils.py index 52c6289c..1df819e8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,11 +5,11 @@ import socket import string import sys +import pathlib from typing import Type from docker.models.containers import Container import httpretty -import py import pytest from universum import submit, poll, main, github_handler, nonci, __main__ @@ -110,8 +110,8 @@ def create_empty_settings(test_type: str) -> ModuleNamespace: class BaseVcsClient: def __init__(self) -> None: - self.root_directory: py.path.local - self.repo_file: py.path.local + self.root_directory: pathlib.Path + self.repo_file: pathlib.Path def get_last_change(self): pass @@ -191,8 +191,8 @@ def assert_request_body_contained(key, value): class BaseTestEnvironment: - def __init__(self, client: BaseVcsClient, directory: py.path.local, test_type: str, db_file: str): - self.temp_dir: py.path.local = directory + def __init__(self, client: BaseVcsClient, directory: pathlib.Path, test_type: str, db_file: str): + self.temp_dir: pathlib.Path = directory self.vcs_client = client self.settings: ModuleNamespace = create_empty_settings(test_type) @@ -200,22 +200,26 @@ def __init__(self, client: BaseVcsClient, directory: py.path.local, test_type: s self.settings.Poll.db_file = db_file self.settings.JenkinsServerForTrigger.trigger_url = "https://localhost/?cl=%s" self.settings.AutomationServer.type = "jenkins" - self.settings.ProjectDirectory.project_root = str(self.temp_dir.mkdir("project_root")) + project_root_dir = self.temp_dir / "project_root" + project_root_dir.mkdir() + self.settings.ProjectDirectory.project_root = str(project_root_dir) elif test_type == "submit": self.settings.Submit.commit_message = "Test CL" # For submitter, the main working dir (project_root) should be the root # of the VCS workspace/client self.settings.ProjectDirectory.project_root = str(self.vcs_client.root_directory) elif test_type in ("main", "nonci"): - self.configs_file = self.temp_dir.join("configs.py") - self.configs_file.write(simple_test_config) + self.configs_file = self.temp_dir / "configs.py" + self.configs_file.write_text(simple_test_config) self.settings.Launcher.config_path = str(self.configs_file) - self.artifact_dir = self.temp_dir.mkdir("artifacts") + self.artifact_dir = self.temp_dir / "artifacts" + self.artifact_dir.mkdir() self.settings.ArtifactCollector.artifact_dir = str(self.artifact_dir) # The project_root directory must not exist before launching main - self.settings.ProjectDirectory.project_root = str(self.temp_dir.join("project_root")) + self.settings.ProjectDirectory.project_root = str(self.temp_dir / "project_root") if test_type == "nonci": - self.temp_dir.mkdir("project_root") + nonci_dir = self.temp_dir / "project_root" + nonci_dir.mkdir() self.settings.Launcher.output = "console" self.settings.AutomationServer.type = "local" self.settings.Output.type = "term" @@ -257,5 +261,6 @@ def __init__(self, directory, test_type): if test_type != "nonci": self.settings.Vcs.type = "none" if test_type == "main": - self.src_dir = directory.ensure_dir('project_sources') + self.src_dir = directory / 'project_sources' + self.src_dir.mkdir() self.settings.LocalMainVcs.source_dir = str(self.src_dir) diff --git a/universum/analyzers/uncrustify.py b/universum/analyzers/uncrustify.py index 0d3cf3be..a5bc6d28 100755 --- a/universum/analyzers/uncrustify.py +++ b/universum/analyzers/uncrustify.py @@ -2,7 +2,7 @@ import difflib import os import shutil -from pathlib import Path +import pathlib from typing import Callable, List, Optional, Tuple @@ -31,19 +31,19 @@ def main(settings: argparse.Namespace) -> List[utils.ReportData]: if not settings.cfg_file and 'UNCRUSTIFY_CONFIG' not in os.environ: raise EnvironmentError("Please specify the '--cfg_file' parameter " "or set an env. variable 'UNCRUSTIFY_CONFIG'") - target_folder: Path = utils.normalize(settings.output_directory) - if target_folder.exists() and target_folder.samefile(Path.cwd()): + target_folder: pathlib.Path = utils.normalize(settings.output_directory) + if target_folder.exists() and target_folder.samefile(pathlib.Path.cwd()): raise EnvironmentError("Target and source folders for uncrustify are not allowed to match") - html_diff_file_writer: Optional[Callable[[Path, List[str], List[str]], None]] = None + html_diff_file_writer: Optional[Callable[[pathlib.Path, List[str], List[str]], None]] = None if settings.write_html: wrapcolumn, tabsize = _get_wrapcolumn_tabsize(settings.cfg_file) html_diff_file_writer = HtmlDiffFileWriter(target_folder, wrapcolumn, tabsize) - files: List[Tuple[Path, Path]] = [] + files: List[Tuple[pathlib.Path, pathlib.Path]] = [] for src_file in settings.file_list: src_file_absolute = utils.normalize(src_file) - src_file_relative = src_file_absolute.relative_to(Path.cwd()) - target_file_absolute: Path = target_folder.joinpath(src_file_relative) + src_file_relative = src_file_absolute.relative_to(pathlib.Path.cwd()) + target_file_absolute: pathlib.Path = target_folder.joinpath(src_file_relative) files.append((src_file_absolute, target_file_absolute)) cmd = ["uncrustify", "-q", "-c", settings.cfg_file, "--prefix", settings.output_directory] cmd.extend(settings.file_list) @@ -53,19 +53,19 @@ def main(settings: argparse.Namespace) -> List[utils.ReportData]: class HtmlDiffFileWriter: - def __init__(self, target_folder: Path, wrapcolumn: int, tabsize: int) -> None: + def __init__(self, target_folder: pathlib.Path, wrapcolumn: int, tabsize: int) -> None: self.target_folder = target_folder self.differ = difflib.HtmlDiff(tabsize=tabsize, wrapcolumn=wrapcolumn) - def __call__(self, file: Path, src: List[str], target: List[str]) -> None: - file_relative = file.relative_to(Path.cwd()) + def __call__(self, file: pathlib.Path, src: List[str], target: List[str]) -> None: + file_relative = file.relative_to(pathlib.Path.cwd()) out_file_name: str = str(file_relative).replace('/', '_') + '.html' with open(self.target_folder.joinpath(out_file_name), 'w', encoding="utf-8") as out_file: out_file.write(self.differ.make_file(src, target, context=False)) -def uncrustify_output_parser(files: List[Tuple[Path, Path]], - write_diff_file: Optional[Callable[[Path, List[str], List[str]], None]] +def uncrustify_output_parser(files: List[Tuple[pathlib.Path, pathlib.Path]], + write_diff_file: Optional[Callable[[pathlib.Path, List[str], List[str]], None]] ) -> List[utils.ReportData]: result: List[utils.ReportData] = [] for src_file, uncrustify_file in files: @@ -91,7 +91,7 @@ def _get_wrapcolumn_tabsize(cfg_file: str) -> Tuple[int, int]: return wrapcolumn, tabsize -def _get_issues_from_diff(src_file: Path, src: List[str], target: List[str]) -> List[utils.ReportData]: +def _get_issues_from_diff(src_file: pathlib.Path, src: List[str], target: List[str]) -> List[utils.ReportData]: result = [] matching_blocks: List[difflib.Match] = \ difflib.SequenceMatcher(a=src, b=target).get_matching_blocks() @@ -102,7 +102,7 @@ def _get_issues_from_diff(src_file: Path, src: List[str], target: List[str]) -> if not block: continue line, before, after = block - path: Path = src_file.relative_to(Path.cwd()) + path: pathlib.Path = src_file.relative_to(pathlib.Path.cwd()) message = _get_issue_message(before, after) result.append(utils.ReportData( symbol="Uncrustify Code Style issue", diff --git a/universum/config_creator.py b/universum/config_creator.py index f2f0c53d..a9fbfa70 100644 --- a/universum/config_creator.py +++ b/universum/config_creator.py @@ -1,4 +1,4 @@ -from pathlib import Path +import pathlib import sys from .modules.output.output import MinimalOut @@ -19,7 +19,7 @@ def execute(self) -> None: config_name = ".universum.py" self.out.log(f"Creating an example configuration file '{config_name}'") - config = Path(config_name) + config = pathlib.Path(config_name) config.write_text(f"""#!/usr/bin/env {PYTHON_VERSION} from universum.configuration_support import Configuration, Step From 041aad3f5537feea6bf7ad197745449d3d663ba4 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Fri, 3 Feb 2023 09:27:42 +0200 Subject: [PATCH 07/32] fix(checks): ignore Pylint warnings for better readability --- pylintrc | 3 ++- universum/modules/output/terminal_based_output.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pylintrc b/pylintrc index 580f2ef8..7b39d854 100644 --- a/pylintrc +++ b/pylintrc @@ -20,7 +20,8 @@ disable = missing-docstring, no-member, unsupported-assignment-operation, super-init-not-called, - wildcard-import + wildcard-import, + use-dict-literal [BASIC] no-docstring-rgx = .* diff --git a/universum/modules/output/terminal_based_output.py b/universum/modules/output/terminal_based_output.py index f4004d01..980340c4 100644 --- a/universum/modules/output/terminal_based_output.py +++ b/universum/modules/output/terminal_based_output.py @@ -23,7 +23,7 @@ class TerminalBasedOutput(BaseOutput): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.block_level = 0 - self.unicode_acceptable = (locale.getpreferredencoding() == "UTF-8") + self.unicode_acceptable = (locale.getpreferredencoding() == "UTF-8") # pylint: disable = superfluous-parens @staticmethod def _stdout(*args, **kwargs) -> None: From 3d80c6a2d3ce98f989d4f48e738d1163d81b5fc1 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Fri, 3 Feb 2023 10:02:14 +0200 Subject: [PATCH 08/32] test(artifact_collector): preprocessing artifacts in the 'artifacts' directory (#756) --- tests/test_preprocess_artifacts.py | 36 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index 6d067569..676d8241 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -39,8 +39,8 @@ def check_step_artifact_present(self) -> None: def check_step_artifact_absent(self) -> None: assert not os.path.exists(self.artifact_path) - def create_artifact_file(self) -> None: - precreated_artifact: pathlib.Path = self.src_dir / self.artifact_name + def create_artifact_file(self, directory: pathlib.Path) -> None: + precreated_artifact: pathlib.Path = directory / self.artifact_name with open(precreated_artifact, "w", encoding="utf-8") as f: f.write("pre-created artifact content") @@ -50,29 +50,35 @@ def test_env(tmp_path: pathlib.Path) -> Generator[ArtifactsTestEnvironment, None yield ArtifactsTestEnvironment(tmp_path) -def test_no_artifact_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: - test_env.write_config_file(artifact_prebuild_clean=True) +@pytest.mark.parametrize("prebuild_clean", [True, False]) +def test_no_artifact(test_env: ArtifactsTestEnvironment, + prebuild_clean: bool) -> None: + test_env.write_config_file(artifact_prebuild_clean=prebuild_clean) test_env.run() test_env.check_step_artifact_present() -def test_no_artifact_no_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: - test_env.write_config_file(artifact_prebuild_clean=False) - test_env.run() - test_env.check_step_artifact_present() - - -def test_existing_artifact_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: +def test_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: test_env.write_config_file(artifact_prebuild_clean=True) - test_env.create_artifact_file() + test_env.create_artifact_file(test_env.src_dir) test_env.run() test_env.check_step_artifact_present() -def test_existing_artifact_no_prebuild_clean(test_env: ArtifactsTestEnvironment, - stdout_checker: FuzzyCallChecker) -> None: +def test_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironment, + stdout_checker: FuzzyCallChecker) -> None: test_env.write_config_file(artifact_prebuild_clean=False) - test_env.create_artifact_file() + test_env.create_artifact_file(test_env.src_dir) test_env.run(expect_failure=True) stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) test_env.check_step_artifact_absent() + + +@pytest.mark.parametrize("prebuild_clean", [True, False]) +def test_artifact_in_artifacts_dir(test_env: ArtifactsTestEnvironment, + stdout_checker: FuzzyCallChecker, + prebuild_clean: bool) -> None: + test_env.write_config_file(artifact_prebuild_clean=prebuild_clean) + test_env.create_artifact_file(test_env.artifact_dir) + test_env.run(expect_failure=True) + stdout_checker.assert_has_calls_with_param("already present in artifact directory") From 50a7aaf680747e43f097362922637b4f5b34ca38 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Tue, 7 Feb 2023 10:15:59 +0200 Subject: [PATCH 09/32] test(artifact_collector): preprocessing directory artifacts (#759) --- tests/test_preprocess_artifacts.py | 80 ++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index 676d8241..af191b16 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -3,6 +3,7 @@ import inspect import os import pathlib +import zipfile from typing import Generator import pytest @@ -18,31 +19,57 @@ def __init__(self, tmp_path: pathlib.Path) -> None: self.artifact_name: str = "artifact" self.artifact_path: pathlib.Path = self.artifact_dir / self.artifact_name self.artifact_content: str = "artifact content" + self.dir_name: str = "artifacts_test_dir" + self.dir_archive: pathlib.Path = self.artifact_dir / f"{self.dir_name}.zip" + self.artifact_in_dir: pathlib.Path = self.artifact_dir / self.dir_name / self.artifact_name def write_config_file(self, artifact_prebuild_clean: bool) -> None: + artifact_in_dir = f"{self.dir_name}/{self.artifact_name}" config: str = inspect.cleandoc(f""" from universum.configuration_support import Configuration, Step - step = Step(name='Step', - command=['bash', '-c', 'echo "{self.artifact_content}" > {self.artifact_name}'], - artifacts='{self.artifact_name}', - artifact_prebuild_clean={artifact_prebuild_clean}) - configs = Configuration([step]) + step_with_file = Step( + name='Step with file', + command=['bash', '-c', 'echo "{self.artifact_content}" > {self.artifact_name}'], + artifacts='{self.artifact_name}', + artifact_prebuild_clean={artifact_prebuild_clean}) + step_with_dir = Step( + name='Step with directory', + command=['bash', '-c', 'mkdir {self.dir_name}; echo "{self.artifact_content}" > {artifact_in_dir}'], + artifacts='{self.dir_name}', + artifact_prebuild_clean={artifact_prebuild_clean}) + configs = Configuration([step_with_file, step_with_dir]) """) self.configs_file.write_text(config, "utf-8") - def check_step_artifact_present(self) -> None: - assert os.path.exists(self.artifact_path) - with open(self.artifact_path, encoding="utf-8") as f: + def create_artifact_file(self, directory: pathlib.Path) -> None: + precreated_artifact: pathlib.Path = directory / self.artifact_name + with open(precreated_artifact, "w", encoding="utf-8") as f: + f.write("pre-created artifact content") + + def create_artifacts_dir(self, directory: pathlib.Path) -> None: + precreated_artifacts_dir: pathlib.Path = directory / self.dir_name + precreated_artifacts_dir.mkdir() + self.create_artifact_file(precreated_artifacts_dir) + + def check_step_artifact_present(self, path: pathlib.Path) -> None: + assert os.path.exists(path) + with open(path, encoding="utf-8") as f: content: str = f.read().replace("\n", "") assert content == self.artifact_content def check_step_artifact_absent(self) -> None: assert not os.path.exists(self.artifact_path) - def create_artifact_file(self, directory: pathlib.Path) -> None: - precreated_artifact: pathlib.Path = directory / self.artifact_name - with open(precreated_artifact, "w", encoding="utf-8") as f: - f.write("pre-created artifact content") + def check_step_dir_zip_artifact_present(self) -> None: + assert os.path.exists(self.dir_archive) + with zipfile.ZipFile(self.dir_archive) as dir_zip: + assert self.artifact_name in dir_zip.namelist() + with dir_zip.open(self.artifact_name) as f: + content: str = f.read().decode(encoding="utf-8").replace("\n", "") + assert content == self.artifact_content + + def check_step_dir_artifact_absent(self) -> None: + assert not os.path.exists(self.dir_archive) @pytest.fixture() @@ -55,14 +82,14 @@ def test_no_artifact(test_env: ArtifactsTestEnvironment, prebuild_clean: bool) -> None: test_env.write_config_file(artifact_prebuild_clean=prebuild_clean) test_env.run() - test_env.check_step_artifact_present() + test_env.check_step_artifact_present(test_env.artifact_path) def test_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: test_env.write_config_file(artifact_prebuild_clean=True) test_env.create_artifact_file(test_env.src_dir) test_env.run() - test_env.check_step_artifact_present() + test_env.check_step_artifact_present(test_env.artifact_path) def test_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironment, @@ -74,6 +101,31 @@ def test_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironmen test_env.check_step_artifact_absent() +def test_dir_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: + test_env.write_config_file(artifact_prebuild_clean=True) + test_env.create_artifacts_dir(test_env.src_dir) + test_env.settings.ArtifactCollector.no_archive = False + test_env.run() + test_env.check_step_dir_zip_artifact_present() + + +def test_dir_artifact_in_sources_prebuild_clean_no_archive(test_env: ArtifactsTestEnvironment) -> None: + test_env.write_config_file(artifact_prebuild_clean=True) + test_env.create_artifacts_dir(test_env.src_dir) + test_env.settings.ArtifactCollector.no_archive = True + test_env.run() + test_env.check_step_artifact_present(test_env.artifact_in_dir) + + +def test_dir_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironment, + stdout_checker: FuzzyCallChecker) -> None: + test_env.write_config_file(artifact_prebuild_clean=False) + test_env.create_artifacts_dir(test_env.src_dir) + test_env.run(expect_failure=True) + stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) + test_env.check_step_dir_artifact_absent() + + @pytest.mark.parametrize("prebuild_clean", [True, False]) def test_artifact_in_artifacts_dir(test_env: ArtifactsTestEnvironment, stdout_checker: FuzzyCallChecker, From 13f352ab8e843784709a60286c6348d2b4514b0f Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Tue, 7 Feb 2023 17:38:49 +0200 Subject: [PATCH 10/32] fix(report): set exit code even if reporting crashes 1. Move 'fail_unsuccessful' option that changes program behavior to Main module 2. Change 'pass_errors=False' in result reporting to explicit exception handling for the guaranteed return value 3. Fix bug caused by p.2 4. Fix typing warning --- tests/test_regression.py | 20 ++++++++++++++++++++ universum/main.py | 6 +++++- universum/modules/reporter.py | 30 ++++++++++++++++-------------- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/tests/test_regression.py b/tests/test_regression.py index cc9223f4..0cf43304 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -226,3 +226,23 @@ def test_success_p4_resolve_unshelved_multiple(perforce_environment_with_files: repo_state = (env.artifact_dir / 'REPOSITORY_STATE.txt').read_text() assert p4_files[0].name in repo_state assert p4_files[1].name in repo_state + + +def test_exit_code_failed_report(perforce_environment: P4TestEnvironment): + """ + This test checks for previous bug where exceptions during result reporting led to exit code 0 + even when '--fail-unsuccessful' option was enabled (and some steps failed) + """ + config = """ +from universum.configuration_support import Configuration + +configs = Configuration([dict(name="Unsuccessful step", command=["exit", "1"])]) +""" + perforce_environment.shelve_config(config) + perforce_environment.settings.MainVcs.report_to_review = True + perforce_environment.settings.Swarm.server_url = "some_server" + perforce_environment.settings.Swarm.review_id = "some_id" + perforce_environment.settings.Swarm.change = perforce_environment.settings.PerforceMainVcs.shelve_cls[0] + perforce_environment.settings.Main.fail_unsuccessful = True + + perforce_environment.run(expect_failure=True) diff --git a/universum/main.py b/universum/main.py index e356da5b..011c934b 100644 --- a/universum/main.py +++ b/universum/main.py @@ -43,6 +43,8 @@ def define_arguments(argument_parser: ModuleArgumentParser) -> None: help="Only applies to build steps where ``code_report=True``; " "disables calculating analysis diff for changed files, " "in this case full analysis report will be published") + argument_parser.add_argument("--fail-unsuccessful", action="store_true", dest="fail_unsuccessful", + help="Return non-zero exit code if any step failed") def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -88,7 +90,9 @@ def execute(self) -> None: self.code_report_collector.repo_diff = repo_diff self.code_report_collector.report_code_report_results() self.artifacts.report_artifacts() - self.reporter.report_build_result() + result = self.reporter.report_build_result() + if self.settings.fail_unsuccessful and not result: + raise SilentAbortException(1) def finalize(self) -> None: if self.settings.no_finalize: diff --git a/universum/modules/reporter.py b/universum/modules/reporter.py index 0d3c95c2..7819bfab 100644 --- a/universum/modules/reporter.py +++ b/universum/modules/reporter.py @@ -1,12 +1,11 @@ from collections import defaultdict from typing import Dict, List, Tuple - from typing_extensions import TypedDict from . import automation_server from .output import HasOutput from .structure_handler import HasStructure, Block -from ..lib.ci_exception import SilentAbortException +from ..lib.ci_exception import CiException from ..lib.gravity import Dependency from ..lib.utils import make_block @@ -54,8 +53,6 @@ def define_arguments(argument_parser): help="Include only the short list of failed steps to reporting comments") parser.add_argument("--report-no-vote", "-rnv", action="store_true", dest="no_vote", help="Do not vote up/down review depending on result") - parser.add_argument("--fail-unsuccessful", "-rfu", action="store_true", dest="fail_unsuccessful", - help="Return non-zero exit code if any step failed") def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -101,11 +98,10 @@ def report_artifacts(self, artifact_list): def code_report(self, path: str, message: ReportMessage) -> None: self.code_report_comments[path].append(message) - @make_block("Reporting build result", pass_errors=False) - def report_build_result(self): + def _report_build_result(self) -> bool: if self.report_initialized is False: self.out.log("Not reporting: no build steps executed") - return + return False if self.settings.only_fails_short: self.settings.only_fails = True @@ -122,9 +118,7 @@ def report_build_result(self): if not self.observers: self.out.log("Nowhere to report. Skipping...") - if self.settings.fail_unsuccessful and not is_successful: - raise SilentAbortException(1) - return + return is_successful if is_successful: self.out.log("Reporting successful build...") @@ -132,11 +126,11 @@ def report_build_result(self): text = "Sending comment skipped. " + \ "To report build success, use '--report-build-success' option" self.out.log(text) - text = None + text = "" else: self.out.log("Reporting failed build...") - if text is not None: + if text: if not self.settings.report_start: text += "\n\n" + self.automation_server.report_build_location() @@ -154,8 +148,16 @@ def report_build_result(self): for observer in self.observers: observer.code_report_to_review(self.code_report_comments) - if self.settings.fail_unsuccessful and not is_successful: - raise SilentAbortException(1) + return is_successful + + @make_block("Reporting build result") + def report_build_result(self) -> bool: + try: + return self._report_build_result() + except CiException as e: + self.out.log_error(str(e)) + self.structure.fail_current_block() + return False def _report_steps_recursively(self, block: Block, text: str, indent: str) -> Tuple[str, bool]: has_children: bool = bool(block.children) From 58d0abea32c5ffe5877c5a42b9d7c97301458bba Mon Sep 17 00:00:00 2001 From: dogbert911 <98908967+dogbert911@users.noreply.github.com> Date: Thu, 9 Feb 2023 08:48:53 +0200 Subject: [PATCH 11/32] fix(swarm): get review version from pass_link --- universum/modules/vcs/swarm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/universum/modules/vcs/swarm.py b/universum/modules/vcs/swarm.py index 6c7d30ce..53eb8e46 100644 --- a/universum/modules/vcs/swarm.py +++ b/universum/modules/vcs/swarm.py @@ -131,7 +131,7 @@ def update_review_version(self): if self.review_version: return if self.settings.fail_link: - self.review_version = get_version_from_link(self.settings.pass_link) + self.review_version = get_version_from_link(self.settings.fail_link) if self.review_version: return self.out.log("PASS/FAIL links either missing or have unexpected format; " From 70941fa225faa65e5a14e20c20477c9bf1af76c8 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Thu, 9 Feb 2023 09:00:10 +0200 Subject: [PATCH 12/32] test(artifact_collector): parametrize 'test_dir_artifact_in_sources_prebuild_clean' test (#761) --- tests/test_preprocess_artifacts.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index af191b16..cef1c35d 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -4,7 +4,7 @@ import os import pathlib import zipfile -from typing import Generator +from typing import Generator, Callable import pytest @@ -77,6 +77,15 @@ def test_env(tmp_path: pathlib.Path) -> Generator[ArtifactsTestEnvironment, None yield ArtifactsTestEnvironment(tmp_path) +class ArtifactsTestData: + no_archive: bool + artifact_check_func: Callable[[ArtifactsTestEnvironment], None] + + def __init__(self, no_archive, artifact_check_func) -> None: + self.no_archive = no_archive + self.artifact_check_func = artifact_check_func + + @pytest.mark.parametrize("prebuild_clean", [True, False]) def test_no_artifact(test_env: ArtifactsTestEnvironment, prebuild_clean: bool) -> None: @@ -101,20 +110,16 @@ def test_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironmen test_env.check_step_artifact_absent() -def test_dir_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: - test_env.write_config_file(artifact_prebuild_clean=True) - test_env.create_artifacts_dir(test_env.src_dir) - test_env.settings.ArtifactCollector.no_archive = False - test_env.run() - test_env.check_step_dir_zip_artifact_present() - - -def test_dir_artifact_in_sources_prebuild_clean_no_archive(test_env: ArtifactsTestEnvironment) -> None: +@pytest.mark.parametrize("test_data", + [ArtifactsTestData(False, lambda env: env.check_step_dir_zip_artifact_present()), + ArtifactsTestData(True, lambda env: env.check_step_artifact_present(env.artifact_in_dir))]) +def test_dir_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment, + test_data: ArtifactsTestData) -> None: test_env.write_config_file(artifact_prebuild_clean=True) test_env.create_artifacts_dir(test_env.src_dir) - test_env.settings.ArtifactCollector.no_archive = True + test_env.settings.ArtifactCollector.no_archive = test_data.no_archive test_env.run() - test_env.check_step_artifact_present(test_env.artifact_in_dir) + test_data.artifact_check_func(test_env) def test_dir_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironment, From 3456a758920916f34925bdb34463d1f8a427a1dc Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Fri, 10 Feb 2023 16:24:09 +0200 Subject: [PATCH 13/32] test(artifact_collector): minor tests refactoring (#765) * remove 'step' from function names * remove 'os' module usage * simplify test parametrization --- tests/test_preprocess_artifacts.py | 47 ++++++++++++------------------ 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index cef1c35d..1d8a227c 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -1,7 +1,6 @@ # pylint: disable = redefined-outer-name import inspect -import os import pathlib import zipfile from typing import Generator, Callable @@ -51,25 +50,25 @@ def create_artifacts_dir(self, directory: pathlib.Path) -> None: precreated_artifacts_dir.mkdir() self.create_artifact_file(precreated_artifacts_dir) - def check_step_artifact_present(self, path: pathlib.Path) -> None: - assert os.path.exists(path) + def check_artifact_present(self, path: pathlib.Path) -> None: + assert path.exists() with open(path, encoding="utf-8") as f: content: str = f.read().replace("\n", "") assert content == self.artifact_content - def check_step_artifact_absent(self) -> None: - assert not os.path.exists(self.artifact_path) + def check_artifact_absent(self) -> None: + assert not self.artifact_path.exists() - def check_step_dir_zip_artifact_present(self) -> None: - assert os.path.exists(self.dir_archive) + def check_dir_zip_artifact_present(self) -> None: + assert self.dir_archive.exists() with zipfile.ZipFile(self.dir_archive) as dir_zip: assert self.artifact_name in dir_zip.namelist() with dir_zip.open(self.artifact_name) as f: content: str = f.read().decode(encoding="utf-8").replace("\n", "") assert content == self.artifact_content - def check_step_dir_artifact_absent(self) -> None: - assert not os.path.exists(self.dir_archive) + def check_dir_artifact_absent(self) -> None: + assert not self.dir_archive.exists() @pytest.fixture() @@ -77,28 +76,19 @@ def test_env(tmp_path: pathlib.Path) -> Generator[ArtifactsTestEnvironment, None yield ArtifactsTestEnvironment(tmp_path) -class ArtifactsTestData: - no_archive: bool - artifact_check_func: Callable[[ArtifactsTestEnvironment], None] - - def __init__(self, no_archive, artifact_check_func) -> None: - self.no_archive = no_archive - self.artifact_check_func = artifact_check_func - - @pytest.mark.parametrize("prebuild_clean", [True, False]) def test_no_artifact(test_env: ArtifactsTestEnvironment, prebuild_clean: bool) -> None: test_env.write_config_file(artifact_prebuild_clean=prebuild_clean) test_env.run() - test_env.check_step_artifact_present(test_env.artifact_path) + test_env.check_artifact_present(test_env.artifact_path) def test_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: test_env.write_config_file(artifact_prebuild_clean=True) test_env.create_artifact_file(test_env.src_dir) test_env.run() - test_env.check_step_artifact_present(test_env.artifact_path) + test_env.check_artifact_present(test_env.artifact_path) def test_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironment, @@ -107,19 +97,20 @@ def test_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironmen test_env.create_artifact_file(test_env.src_dir) test_env.run(expect_failure=True) stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) - test_env.check_step_artifact_absent() + test_env.check_artifact_absent() -@pytest.mark.parametrize("test_data", - [ArtifactsTestData(False, lambda env: env.check_step_dir_zip_artifact_present()), - ArtifactsTestData(True, lambda env: env.check_step_artifact_present(env.artifact_in_dir))]) +@pytest.mark.parametrize("no_archive", [False, True]) def test_dir_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment, - test_data: ArtifactsTestData) -> None: + no_archive: bool) -> None: test_env.write_config_file(artifact_prebuild_clean=True) test_env.create_artifacts_dir(test_env.src_dir) - test_env.settings.ArtifactCollector.no_archive = test_data.no_archive + test_env.settings.ArtifactCollector.no_archive = no_archive test_env.run() - test_data.artifact_check_func(test_env) + if no_archive: + test_env.check_artifact_present(test_env.artifact_in_dir) + else: + test_env.check_dir_zip_artifact_present() def test_dir_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironment, @@ -128,7 +119,7 @@ def test_dir_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnviro test_env.create_artifacts_dir(test_env.src_dir) test_env.run(expect_failure=True) stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) - test_env.check_step_dir_artifact_absent() + test_env.check_dir_artifact_absent() @pytest.mark.parametrize("prebuild_clean", [True, False]) From a949f77e765dcb3545e2d638f02a0747fc5bc03e Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Mon, 27 Feb 2023 16:35:32 +0200 Subject: [PATCH 14/32] fix(artifact_collector): different exception messages for zip and non-zip artifact (#767) * extend tests * update exception message * update Step description * update 'note' formatting * fix 'note' --- tests/test_preprocess_artifacts.py | 24 +++++++++++++++--------- universum/configuration_support.py | 6 ++++++ universum/modules/artifact_collector.py | 18 ++++++++++++------ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index 1d8a227c..c5714540 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -40,15 +40,16 @@ def write_config_file(self, artifact_prebuild_clean: bool) -> None: """) self.configs_file.write_text(config, "utf-8") - def create_artifact_file(self, directory: pathlib.Path) -> None: - precreated_artifact: pathlib.Path = directory / self.artifact_name + def create_artifact_file(self, directory: pathlib.Path, file_name: str, is_zip: bool = False) -> None: + artifact_name: str = f"{file_name}.zip" if is_zip else file_name + precreated_artifact: pathlib.Path = directory / artifact_name with open(precreated_artifact, "w", encoding="utf-8") as f: f.write("pre-created artifact content") def create_artifacts_dir(self, directory: pathlib.Path) -> None: precreated_artifacts_dir: pathlib.Path = directory / self.dir_name precreated_artifacts_dir.mkdir() - self.create_artifact_file(precreated_artifacts_dir) + self.create_artifact_file(precreated_artifacts_dir, self.artifact_name) def check_artifact_present(self, path: pathlib.Path) -> None: assert path.exists() @@ -86,7 +87,7 @@ def test_no_artifact(test_env: ArtifactsTestEnvironment, def test_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: test_env.write_config_file(artifact_prebuild_clean=True) - test_env.create_artifact_file(test_env.src_dir) + test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name) test_env.run() test_env.check_artifact_present(test_env.artifact_path) @@ -94,7 +95,7 @@ def test_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment) def test_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironment, stdout_checker: FuzzyCallChecker) -> None: test_env.write_config_file(artifact_prebuild_clean=False) - test_env.create_artifact_file(test_env.src_dir) + test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name) test_env.run(expect_failure=True) stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) test_env.check_artifact_absent() @@ -122,11 +123,16 @@ def test_dir_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnviro test_env.check_dir_artifact_absent() +@pytest.mark.parametrize("is_zip", [True, False]) +@pytest.mark.parametrize("is_dir", [True, False]) @pytest.mark.parametrize("prebuild_clean", [True, False]) -def test_artifact_in_artifacts_dir(test_env: ArtifactsTestEnvironment, - stdout_checker: FuzzyCallChecker, - prebuild_clean: bool) -> None: +def test_zip_artifact_in_artifacts_dir(test_env: ArtifactsTestEnvironment, + stdout_checker: FuzzyCallChecker, + is_zip: bool, + is_dir: bool, + prebuild_clean: bool) -> None: test_env.write_config_file(artifact_prebuild_clean=prebuild_clean) - test_env.create_artifact_file(test_env.artifact_dir) + artifact_name: str = test_env.dir_name if is_dir else test_env.artifact_name + test_env.create_artifact_file(test_env.artifact_dir, artifact_name, is_zip) test_env.run(expect_failure=True) stdout_checker.assert_has_calls_with_param("already present in artifact directory") diff --git a/universum/configuration_support.py b/universum/configuration_support.py index c911d560..f5706e06 100644 --- a/universum/configuration_support.py +++ b/universum/configuration_support.py @@ -69,6 +69,12 @@ class Step: execution will not proceed. If no required artifacts were found in the end of the `Universum` run, it is also considered a failure. In case of shell-style patterns build is failed if no files or directories matching pattern are found. + + .. note:: + + Universum checks both the artifact itself and an archive with the same name, because it does not know + in advance whether the step is going to create a file or a directory. + report_artifacts Path to the special artifacts for reporting (e.g. to Swarm). Unlike `artifacts` key, `report_artifacts` are not obligatory and their absence is not considered a build failure. A directory cannot be stored diff --git a/universum/modules/artifact_collector.py b/universum/modules/artifact_collector.py index f37d7df9..d72ad53b 100644 --- a/universum/modules/artifact_collector.py +++ b/universum/modules/artifact_collector.py @@ -143,12 +143,11 @@ def preprocess_artifact_list(self, artifact_list, ignore_already_existing=False) raise CriticalCiException(text) # Check existence in 'artifacts' directory: wildcards NOT applied - path_to_check1 = os.path.join(self.artifact_dir, os.path.basename(item["path"])) - path_to_check2 = os.path.join(path_to_check1 + ".zip") - if os.path.exists(path_to_check1) or os.path.exists(path_to_check2): - text = f"Build artifact '{os.path.basename(item['path'])}' already present in artifact directory." - text += "\nPossible reason of this error: previous build results in working directory" - raise CriticalCiException(text) + artifact_file = os.path.join(self.artifact_dir, os.path.basename(item["path"])) + self._check_artifact_absent(artifact_file) + + artifact_zip_archive = os.path.join(artifact_file + ".zip") + self._check_artifact_absent(artifact_zip_archive) @make_block("Preprocessing artifact lists") def set_and_clean_artifacts(self, project_configs: Configuration, ignore_existing_artifacts: bool = False) -> None: @@ -227,3 +226,10 @@ def clean_artifacts_silently(self): pass os.makedirs(self.artifact_dir) self.html_output.artifact_dir_ready = True + + @staticmethod + def _check_artifact_absent(artifact_path: str): + if os.path.exists(artifact_path): + text: str = f"Build artifact '{os.path.basename(artifact_path)}' already present in artifact directory." + text += "\nPossible reason of this error: previous build results in working directory" + raise CriticalCiException(text) From 6a099cec7264317d83b5503267377b658d2154e1 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Wed, 1 Mar 2023 09:30:53 +0200 Subject: [PATCH 15/32] test & fix (#769) --- tests/test_preprocess_artifacts.py | 16 ++++++++++++---- universum/modules/artifact_collector.py | 5 +++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index c5714540..77b68abb 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -3,7 +3,7 @@ import inspect import pathlib import zipfile -from typing import Generator, Callable +from typing import Generator import pytest @@ -68,7 +68,7 @@ def check_dir_zip_artifact_present(self) -> None: content: str = f.read().decode(encoding="utf-8").replace("\n", "") assert content == self.artifact_content - def check_dir_artifact_absent(self) -> None: + def check_dir_zip_artifact_absent(self) -> None: assert not self.dir_archive.exists() @@ -120,13 +120,13 @@ def test_dir_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnviro test_env.create_artifacts_dir(test_env.src_dir) test_env.run(expect_failure=True) stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) - test_env.check_dir_artifact_absent() + test_env.check_dir_zip_artifact_absent() @pytest.mark.parametrize("is_zip", [True, False]) @pytest.mark.parametrize("is_dir", [True, False]) @pytest.mark.parametrize("prebuild_clean", [True, False]) -def test_zip_artifact_in_artifacts_dir(test_env: ArtifactsTestEnvironment, +def test_artifact_in_artifacts_dir(test_env: ArtifactsTestEnvironment, stdout_checker: FuzzyCallChecker, is_zip: bool, is_dir: bool, @@ -136,3 +136,11 @@ def test_zip_artifact_in_artifacts_dir(test_env: ArtifactsTestEnvironment, test_env.create_artifact_file(test_env.artifact_dir, artifact_name, is_zip) test_env.run(expect_failure=True) stdout_checker.assert_has_calls_with_param("already present in artifact directory") + + +def test_zip_artifact_no_archive(test_env: ArtifactsTestEnvironment) -> None: + test_env.settings.ArtifactCollector.no_archive = True + test_env.write_config_file(artifact_prebuild_clean=True) + test_env.create_artifact_file(test_env.artifact_dir, test_env.dir_name, is_zip=True) + test_env.run() + test_env.check_artifact_present(test_env.artifact_in_dir) diff --git a/universum/modules/artifact_collector.py b/universum/modules/artifact_collector.py index d72ad53b..aaf0bba8 100644 --- a/universum/modules/artifact_collector.py +++ b/universum/modules/artifact_collector.py @@ -146,8 +146,9 @@ def preprocess_artifact_list(self, artifact_list, ignore_already_existing=False) artifact_file = os.path.join(self.artifact_dir, os.path.basename(item["path"])) self._check_artifact_absent(artifact_file) - artifact_zip_archive = os.path.join(artifact_file + ".zip") - self._check_artifact_absent(artifact_zip_archive) + if not self.settings.no_archive: + artifact_zip_archive = os.path.join(artifact_file + ".zip") + self._check_artifact_absent(artifact_zip_archive) @make_block("Preprocessing artifact lists") def set_and_clean_artifacts(self, project_configs: Configuration, ignore_existing_artifacts: bool = False) -> None: From 957b90ee2445ff601501ddcfb71f05eefa062dd6 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Fri, 3 Mar 2023 13:36:46 +0200 Subject: [PATCH 16/32] feat(github): add posting to TG via GH actions (#768) * feat(github): add posting to TG via GH actions * fix(github): use existing action for http requests * Update telegram-bot.yml * fix(github): add Universum reports and multiline * fix(github): remove unnecessary steps * fix(github): add escaping, refactor --- .github/workflows/telegram-bot.yml | 53 ++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/telegram-bot.yml diff --git a/.github/workflows/telegram-bot.yml b/.github/workflows/telegram-bot.yml new file mode 100644 index 00000000..02997ce3 --- /dev/null +++ b/.github/workflows/telegram-bot.yml @@ -0,0 +1,53 @@ +name: Telegram bot +on: + pull_request: + types: [opened, synchronize, closed] + issue_comment: + types: [created] + workflow_run: + workflows: [Universum check] + types: [completed] + +jobs: + make-comment: + name: Send comment to TG + runs-on: ubuntu-latest + env: + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + PR_NAME: ${{ github.event.pull_request.title }} + PR_BASE: ${{ github.event.pull_request.base.ref }} + PR_URL: ${{ github.event.pull_request.html_url }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_MERGED: ${{ github.event.pull_request.merged_by.login }} + COMMENT_AUTHOR: ${{ github.event.comment.user.login }} + COMMENT_URL: ${{ github.event.comment.html_url }} + COMMENT_BODY: ${{ github.event.comment.body }} + COMMENT_NUMBER: ${{ github.event.issue.number }} + UNIVERUM_COMMIT: ${{ github.event.workflow_run.head_sha }} + UNIVERSUM_BRANCH: ${{ github.event.workflow_run.head_branch }} + UNIVERSUM_LOG: ${{ github.event.workflow_run.html_url }} + + steps: + - name: Send message to TG + run: | + if [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "opened" ]]; then + ESCAPED_NAME=`echo -e "${{ env.PR_NAME }}" | sed 's/\&/\&/g' | sed 's//\>/g'` + TEXT=`echo -e "${{ env.PR_AUTHOR }} created new PR#${{ env.PR_NUMBER }} '$ESCAPED_NAME' to branch '${{ env.PR_BASE }}'"` + elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "synchronize" ]]; then + TEXT=`echo -e "${{ env.PR_AUTHOR }} updated PR#${{ env.PR_NUMBER }}"` + elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "closed" && "${{ github.event.pull_request.merged }}" == "true" ]]; then + TEXT=`echo -e "${{ env.PR_MERGED }} merged PR#${{ env.PR_NUMBER }} to branch '${{ env.PR_BASE }}'"` + elif [[ ! -z "${{ github.event.comment }}" ]]; then + ESCAPED_TEXT=`echo -e "${{ env.COMMENT_BODY }}"| sed 's/\&/\&/g' | sed 's//\>/g'` + TEXT=`echo -e "${{ env.COMMENT_AUTHOR }} posted the following comment to issue #${{ env.COMMENT_NUMBER }}:\n$ESCAPED_TEXT"` + elif [[ ! -z "${{ github.event.workflow_run }}" && "${{ github.event.workflow_run.conclusion }}" == "success" ]]; then + TEXT=`echo -e "Universum run for branch '${{ env.UNIVERSUM_BRANCH }}' SUCCEDDED; commit ${{ env.UNIVERUM_COMMIT }} "` + elif [[ ! -z "${{ github.event.workflow_run }}" && "${{ github.event.workflow_run.conclusion }}" == "failure" ]]; then + TEXT=`echo -e "Universum run for branch '${{ env.UNIVERSUM_BRANCH }}' FAILED; commit ${{ env.UNIVERUM_COMMIT }} "` + fi + + if [[ ! -z $TEXT ]]; then + curl --get --data-urlencode "chat_id=${{ secrets.TELEGRAM_CHAT_ID }}" --data-urlencode "text=$TEXT" --data-urlencode "parse_mode=HTML" $URL + fi + env: + URL: https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage From 7e9e2b59ec09e96516d2ed898d4771bc89347cab Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Mon, 20 Mar 2023 09:48:27 +0200 Subject: [PATCH 17/32] wildcards_test (#772) --- tests/test_preprocess_artifacts.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index 77b68abb..28947b2e 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -17,6 +17,8 @@ def __init__(self, tmp_path: pathlib.Path) -> None: super().__init__(tmp_path, "main") self.artifact_name: str = "artifact" self.artifact_path: pathlib.Path = self.artifact_dir / self.artifact_name + self.artifact_name_with_suffix = f"{self.artifact_name}_suffix" + self.artifact_path_with_suffix: pathlib.Path = self.artifact_dir / self.artifact_name_with_suffix self.artifact_content: str = "artifact content" self.dir_name: str = "artifacts_test_dir" self.dir_archive: pathlib.Path = self.artifact_dir / f"{self.dir_name}.zip" @@ -38,6 +40,23 @@ def write_config_file(self, artifact_prebuild_clean: bool) -> None: artifact_prebuild_clean={artifact_prebuild_clean}) configs = Configuration([step_with_file, step_with_dir]) """) + self.store_config_to_file(config) + + def write_config_file_wildcard(self, artifact_prebuild_clean: bool) -> None: + config: str = inspect.cleandoc(f""" + from universum.configuration_support import Configuration, Step + step = Step( + name='Step', + command=['bash', '-c', + 'echo "{self.artifact_content}" > {self.artifact_name};' + 'echo "{self.artifact_content}" > {self.artifact_name_with_suffix}'], + artifacts='{self.artifact_name}*', + artifact_prebuild_clean={artifact_prebuild_clean}) + configs = Configuration([step]) + """) + self.store_config_to_file(config) + + def store_config_to_file(self, config: str): self.configs_file.write_text(config, "utf-8") def create_artifact_file(self, directory: pathlib.Path, file_name: str, is_zip: bool = False) -> None: @@ -144,3 +163,12 @@ def test_zip_artifact_no_archive(test_env: ArtifactsTestEnvironment) -> None: test_env.create_artifact_file(test_env.artifact_dir, test_env.dir_name, is_zip=True) test_env.run() test_env.check_artifact_present(test_env.artifact_in_dir) + + +def test_wildcard(test_env: ArtifactsTestEnvironment) -> None: + test_env.write_config_file_wildcard(artifact_prebuild_clean=True) + test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name) + test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name_with_suffix) + test_env.run() + test_env.check_artifact_present(test_env.artifact_path) + test_env.check_artifact_present(test_env.artifact_path_with_suffix) From ddc536736cb78c28edf2acdf5932ebd651b16b8a Mon Sep 17 00:00:00 2001 From: i-keliukh <46344924+i-keliukh@users.noreply.github.com> Date: Mon, 20 Mar 2023 14:30:37 +0200 Subject: [PATCH 18/32] fix(uncrustify): usability issues Fixed the following issues in uncrustify analyzer: - issue with not finding the formatted file if if was passed to the analyzer by using absolute path - issue with not initializing variables leading to exception if their value are not found in uncrustify config file - issue with not handling all variants of the uncrustify config file formatting --- universum/analyzers/uncrustify.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/universum/analyzers/uncrustify.py b/universum/analyzers/uncrustify.py index a5bc6d28..2f24ca2d 100755 --- a/universum/analyzers/uncrustify.py +++ b/universum/analyzers/uncrustify.py @@ -3,6 +3,7 @@ import os import shutil import pathlib +import re from typing import Callable, List, Optional, Tuple @@ -39,14 +40,15 @@ def main(settings: argparse.Namespace) -> List[utils.ReportData]: wrapcolumn, tabsize = _get_wrapcolumn_tabsize(settings.cfg_file) html_diff_file_writer = HtmlDiffFileWriter(target_folder, wrapcolumn, tabsize) + cmd = ["uncrustify", "-q", "-c", settings.cfg_file, "--prefix", settings.output_directory] files: List[Tuple[pathlib.Path, pathlib.Path]] = [] for src_file in settings.file_list: src_file_absolute = utils.normalize(src_file) src_file_relative = src_file_absolute.relative_to(pathlib.Path.cwd()) target_file_absolute: pathlib.Path = target_folder.joinpath(src_file_relative) files.append((src_file_absolute, target_file_absolute)) - cmd = ["uncrustify", "-q", "-c", settings.cfg_file, "--prefix", settings.output_directory] - cmd.extend(settings.file_list) + cmd.append(src_file_relative) + utils.run_for_output(cmd) return uncrustify_output_parser(files, html_diff_file_writer) @@ -82,12 +84,18 @@ def uncrustify_output_parser(files: List[Tuple[pathlib.Path, pathlib.Path]], def _get_wrapcolumn_tabsize(cfg_file: str) -> Tuple[int, int]: + wrapcolumn = 120 + tabsize = 4 with open(cfg_file, encoding="utf-8") as config: for line in config.readlines(): - if line.startswith("code_width"): - wrapcolumn = int(line.split()[2]) - if line.startswith("input_tab_size"): - tabsize = int(line.split()[2]) + match = re.match(r"^\s*([A-Za-z_]+)\s*[,\=]?\s*([0-9]+)\s*$", line) + if not match: + continue + groups = match.groups() + if groups[0] == "code_width": + wrapcolumn = int(groups[1]) + if groups[0] == "input_tab_size": + tabsize = int(groups[1]) return wrapcolumn, tabsize From 3eb7a37872492e39ec06f587b32953a51e7ad43f Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Tue, 21 Mar 2023 13:31:24 +0200 Subject: [PATCH 19/32] test(artifact_collector): parametrize some tests for report_artifacts --- tests/test_preprocess_artifacts.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index 28947b2e..4e1f783b 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -24,19 +24,20 @@ def __init__(self, tmp_path: pathlib.Path) -> None: self.dir_archive: pathlib.Path = self.artifact_dir / f"{self.dir_name}.zip" self.artifact_in_dir: pathlib.Path = self.artifact_dir / self.dir_name / self.artifact_name - def write_config_file(self, artifact_prebuild_clean: bool) -> None: - artifact_in_dir = f"{self.dir_name}/{self.artifact_name}" + def write_config_file(self, artifact_prebuild_clean: bool, is_report_artifact: bool = False) -> None: + artifact_in_dir: str = f"{self.dir_name}/{self.artifact_name}" + artifacts_key: str = "report_artifacts" if is_report_artifact else "artifacts" config: str = inspect.cleandoc(f""" from universum.configuration_support import Configuration, Step step_with_file = Step( name='Step with file', command=['bash', '-c', 'echo "{self.artifact_content}" > {self.artifact_name}'], - artifacts='{self.artifact_name}', + {artifacts_key}='{self.artifact_name}', artifact_prebuild_clean={artifact_prebuild_clean}) step_with_dir = Step( name='Step with directory', command=['bash', '-c', 'mkdir {self.dir_name}; echo "{self.artifact_content}" > {artifact_in_dir}'], - artifacts='{self.dir_name}', + {artifacts_key}='{self.dir_name}', artifact_prebuild_clean={artifact_prebuild_clean}) configs = Configuration([step_with_file, step_with_dir]) """) @@ -104,16 +105,20 @@ def test_no_artifact(test_env: ArtifactsTestEnvironment, test_env.check_artifact_present(test_env.artifact_path) -def test_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment) -> None: - test_env.write_config_file(artifact_prebuild_clean=True) +@pytest.mark.parametrize("is_report_artifact", [True, False]) +def test_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment, + is_report_artifact: bool) -> None: + test_env.write_config_file(artifact_prebuild_clean=True, is_report_artifact=is_report_artifact) test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name) test_env.run() test_env.check_artifact_present(test_env.artifact_path) +@pytest.mark.parametrize("is_report_artifact", [True, False]) def test_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironment, - stdout_checker: FuzzyCallChecker) -> None: - test_env.write_config_file(artifact_prebuild_clean=False) + stdout_checker: FuzzyCallChecker, + is_report_artifact: bool) -> None: + test_env.write_config_file(artifact_prebuild_clean=False, is_report_artifact=is_report_artifact) test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name) test_env.run(expect_failure=True) stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) From 5287747ed8ad538c154d19ae8883dfeaf368d3f7 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Wed, 22 Mar 2023 16:23:08 +0200 Subject: [PATCH 20/32] feat(github): add comments on approve changes Also fix typo and add comment on closed review --- .github/workflows/telegram-bot.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/telegram-bot.yml b/.github/workflows/telegram-bot.yml index 02997ce3..7e291f0b 100644 --- a/.github/workflows/telegram-bot.yml +++ b/.github/workflows/telegram-bot.yml @@ -7,7 +7,9 @@ on: workflow_run: workflows: [Universum check] types: [completed] - + pull_request_review: + types: [submitted, edited, dismissed] + jobs: make-comment: name: Send comment to TG @@ -19,6 +21,8 @@ jobs: PR_URL: ${{ github.event.pull_request.html_url }} PR_NUMBER: ${{ github.event.pull_request.number }} PR_MERGED: ${{ github.event.pull_request.merged_by.login }} + REVIEW_STATE: ${{ github.event.review.state }} + REVIEW_AUTHOR: ${{ github.event.review.user.login }} COMMENT_AUTHOR: ${{ github.event.comment.user.login }} COMMENT_URL: ${{ github.event.comment.html_url }} COMMENT_BODY: ${{ github.event.comment.body }} @@ -30,24 +34,38 @@ jobs: steps: - name: Send message to TG run: | + if [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "opened" ]]; then ESCAPED_NAME=`echo -e "${{ env.PR_NAME }}" | sed 's/\&/\&/g' | sed 's//\>/g'` TEXT=`echo -e "${{ env.PR_AUTHOR }} created new PR#${{ env.PR_NUMBER }} '$ESCAPED_NAME' to branch '${{ env.PR_BASE }}'"` elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "synchronize" ]]; then TEXT=`echo -e "${{ env.PR_AUTHOR }} updated PR#${{ env.PR_NUMBER }}"` + elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "closed" ]]; then + TEXT=`echo -e "${{ env.PR_AUTHOR }} closed PR#${{ env.PR_NUMBER }}"` elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "closed" && "${{ github.event.pull_request.merged }}" == "true" ]]; then TEXT=`echo -e "${{ env.PR_MERGED }} merged PR#${{ env.PR_NUMBER }} to branch '${{ env.PR_BASE }}'"` + elif [[ ! -z "${{ github.event.comment }}" ]]; then ESCAPED_TEXT=`echo -e "${{ env.COMMENT_BODY }}"| sed 's/\&/\&/g' | sed 's//\>/g'` TEXT=`echo -e "${{ env.COMMENT_AUTHOR }} posted the following comment to issue #${{ env.COMMENT_NUMBER }}:\n$ESCAPED_TEXT"` + + elif [[ ! -z "${{ github.event.review }}" && "${{ env.REVIEW_STATE }}" == "changes_requested" ]]; then + TEXT=`echo -e "${{ env.REVIEW_AUTHOR }} requested changes for PR#${{ env.PR_NUMBER }}"` + elif [[ ! -z "${{ github.event.review }}" && "${{ env.REVIEW_STATE }}" != "changes_requested" ]]; then + TEXT=`echo -e "${{ env.REVIEW_AUTHOR }} ${{ env.REVIEW_STATE }} PR#${{ env.PR_NUMBER }}"` + elif [[ -z "${{ github.event.review }}" && "${{ github.event.action }}" == "submitted" ]]; then + TEXT=`echo -e "Due to GitHub Actions bug we cannot identify, who approved PR#${{ env.PR_NUMBER }}"` + elif [[ ! -z "${{ github.event.workflow_run }}" && "${{ github.event.workflow_run.conclusion }}" == "success" ]]; then - TEXT=`echo -e "Universum run for branch '${{ env.UNIVERSUM_BRANCH }}' SUCCEDDED; commit ${{ env.UNIVERUM_COMMIT }} "` + TEXT=`echo -e "Universum run for branch '${{ env.UNIVERSUM_BRANCH }}' SUCCEDED; commit ${{ env.UNIVERUM_COMMIT }} "` elif [[ ! -z "${{ github.event.workflow_run }}" && "${{ github.event.workflow_run.conclusion }}" == "failure" ]]; then TEXT=`echo -e "Universum run for branch '${{ env.UNIVERSUM_BRANCH }}' FAILED; commit ${{ env.UNIVERUM_COMMIT }} "` fi if [[ ! -z $TEXT ]]; then - curl --get --data-urlencode "chat_id=${{ secrets.TELEGRAM_CHAT_ID }}" --data-urlencode "text=$TEXT" --data-urlencode "parse_mode=HTML" $URL + curl --get --data-urlencode "chat_id=${{ secrets.TELEGRAM_CHAT_ID }}" \ + --data-urlencode "text=$TEXT" --data-urlencode "parse_mode=HTML" $URL fi + env: URL: https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage From 30cce1ffff9bb66688c895cea11877963047e2bf Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Wed, 22 Mar 2023 17:03:41 +0200 Subject: [PATCH 21/32] fix(github): wrong if's order --- .github/workflows/telegram-bot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/telegram-bot.yml b/.github/workflows/telegram-bot.yml index 7e291f0b..6d623abc 100644 --- a/.github/workflows/telegram-bot.yml +++ b/.github/workflows/telegram-bot.yml @@ -40,10 +40,10 @@ jobs: TEXT=`echo -e "${{ env.PR_AUTHOR }} created new PR#${{ env.PR_NUMBER }} '$ESCAPED_NAME' to branch '${{ env.PR_BASE }}'"` elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "synchronize" ]]; then TEXT=`echo -e "${{ env.PR_AUTHOR }} updated PR#${{ env.PR_NUMBER }}"` - elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "closed" ]]; then - TEXT=`echo -e "${{ env.PR_AUTHOR }} closed PR#${{ env.PR_NUMBER }}"` elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "closed" && "${{ github.event.pull_request.merged }}" == "true" ]]; then TEXT=`echo -e "${{ env.PR_MERGED }} merged PR#${{ env.PR_NUMBER }} to branch '${{ env.PR_BASE }}'"` + elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "closed" ]]; then + TEXT=`echo -e "${{ env.PR_AUTHOR }} closed PR#${{ env.PR_NUMBER }}"` elif [[ ! -z "${{ github.event.comment }}" ]]; then ESCAPED_TEXT=`echo -e "${{ env.COMMENT_BODY }}"| sed 's/\&/\&/g' | sed 's//\>/g'` From 4556b5bf0a99c322406c240d12d8bd122f61bdec Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Mon, 27 Mar 2023 11:10:14 +0300 Subject: [PATCH 22/32] test(artifact_collector): parametrize some tests for 'nonci' mode --- tests/test_preprocess_artifacts.py | 32 ++++++++++++++++++++---------- tests/utils.py | 6 ++++-- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py index 4e1f783b..bca3fa40 100644 --- a/tests/test_preprocess_artifacts.py +++ b/tests/test_preprocess_artifacts.py @@ -13,8 +13,8 @@ class ArtifactsTestEnvironment(LocalTestEnvironment): - def __init__(self, tmp_path: pathlib.Path) -> None: - super().__init__(tmp_path, "main") + def __init__(self, tmp_path: pathlib.Path, test_type: str) -> None: + super().__init__(tmp_path, test_type) self.artifact_name: str = "artifact" self.artifact_path: pathlib.Path = self.artifact_dir / self.artifact_name self.artifact_name_with_suffix = f"{self.artifact_name}_suffix" @@ -94,7 +94,7 @@ def check_dir_zip_artifact_absent(self) -> None: @pytest.fixture() def test_env(tmp_path: pathlib.Path) -> Generator[ArtifactsTestEnvironment, None, None]: - yield ArtifactsTestEnvironment(tmp_path) + yield ArtifactsTestEnvironment(tmp_path, "main") @pytest.mark.parametrize("prebuild_clean", [True, False]) @@ -105,24 +105,36 @@ def test_no_artifact(test_env: ArtifactsTestEnvironment, test_env.check_artifact_present(test_env.artifact_path) +@pytest.mark.parametrize("test_type", ["main", "nonci"]) @pytest.mark.parametrize("is_report_artifact", [True, False]) -def test_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment, - is_report_artifact: bool) -> None: +def test_artifact_in_sources_prebuild_clean(tmp_path: pathlib.Path, + is_report_artifact: bool, + test_type: str) -> None: + test_env: ArtifactsTestEnvironment = ArtifactsTestEnvironment(tmp_path, test_type) test_env.write_config_file(artifact_prebuild_clean=True, is_report_artifact=is_report_artifact) test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name) test_env.run() test_env.check_artifact_present(test_env.artifact_path) +@pytest.mark.parametrize("test_type", ["main", "nonci"]) @pytest.mark.parametrize("is_report_artifact", [True, False]) -def test_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironment, +def test_artifact_in_sources_no_prebuild_clean(tmp_path: pathlib.Path, stdout_checker: FuzzyCallChecker, - is_report_artifact: bool) -> None: + is_report_artifact: bool, + test_type: str) -> None: + test_env: ArtifactsTestEnvironment = ArtifactsTestEnvironment(tmp_path, test_type) test_env.write_config_file(artifact_prebuild_clean=False, is_report_artifact=is_report_artifact) test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name) - test_env.run(expect_failure=True) - stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) - test_env.check_artifact_absent() + if test_type == "main": + test_env.run(expect_failure=True) + stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) + test_env.check_artifact_absent() + elif test_type == "nonci": + test_env.run() + test_env.check_artifact_present(test_env.artifact_path) + else: + pytest.fail(f"Unexpected type type: {test_type}") @pytest.mark.parametrize("no_archive", [False, True]) diff --git a/tests/utils.py b/tests/utils.py index 1df819e8..47f2f930 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -218,8 +218,8 @@ def __init__(self, client: BaseVcsClient, directory: pathlib.Path, test_type: st # The project_root directory must not exist before launching main self.settings.ProjectDirectory.project_root = str(self.temp_dir / "project_root") if test_type == "nonci": - nonci_dir = self.temp_dir / "project_root" - nonci_dir.mkdir() + self.nonci_dir = self.temp_dir / "project_root" + self.nonci_dir.mkdir() self.settings.Launcher.output = "console" self.settings.AutomationServer.type = "local" self.settings.Output.type = "term" @@ -264,3 +264,5 @@ def __init__(self, directory, test_type): self.src_dir = directory / 'project_sources' self.src_dir.mkdir() self.settings.LocalMainVcs.source_dir = str(self.src_dir) + if test_type == "nonci": + self.src_dir = self.nonci_dir From b75f759c80a0754882426f04705fa4fa404383c4 Mon Sep 17 00:00:00 2001 From: i-keliukh <46344924+i-keliukh@users.noreply.github.com> Date: Tue, 4 Apr 2023 14:31:41 +0300 Subject: [PATCH 23/32] fix(test): new docker engine version compatibility After some update the docker engine started to require passing full paths to bind mounts when starting the contianer. The issue is reproduced on version 23.0.2. The error message from the docker engine is the following: "invalid volume specification: '.work:.work:rw': invalid mount config for type "volume": invalid mount path: '.work' mount path must be absolute". --- tests/deployment_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/deployment_utils.py b/tests/deployment_utils.py index 8d080315..32a67194 100644 --- a/tests/deployment_utils.py +++ b/tests/deployment_utils.py @@ -42,7 +42,8 @@ def add_bind_dirs(self, directories): if self._container: self.request.raiseerror("Container is already running, no dirs can be bound!") for directory in directories: - self._volumes[directory] = {'bind': directory, 'mode': 'rw'} + absolute_dir = str(pathlib.Path(directory).absolute()) + self._volumes[absolute_dir] = {'bind': absolute_dir, 'mode': 'rw'} def add_environment_variables(self, variables): if self._container: From b828b1d6dfbdf094dce4d5f153a335e75a0d8a38 Mon Sep 17 00:00:00 2001 From: i-keliukh <46344924+i-keliukh@users.noreply.github.com> Date: Wed, 5 Apr 2023 20:53:21 +0300 Subject: [PATCH 24/32] fix(analyzer): incorrect program name in help Before fix: $ python3.10 -m universum.analyzers.mypy --help usage: mypy.py [-h] ... After fix: $ python3.10 -m universum.analyzers.mypy --help usage: python3.10 -m universum.analyzers.mypy [-h] ... --- universum/analyzers/mypy.py | 7 ++----- universum/analyzers/pylint.py | 7 ++----- universum/analyzers/uncrustify.py | 11 ++++------- universum/analyzers/utils.py | 24 +++++++++++++++++++----- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/universum/analyzers/mypy.py b/universum/analyzers/mypy.py index d5c95e54..8536c835 100644 --- a/universum/analyzers/mypy.py +++ b/universum/analyzers/mypy.py @@ -1,19 +1,16 @@ import argparse - from typing import List from . import utils -def mypy_argument_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Mypy analyzer") +def add_mypy_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--config-file", dest="config_file", type=str, help="Specify a configuration file.") utils.add_python_version_argument(parser) - return parser @utils.sys_exit -@utils.analyzer(mypy_argument_parser()) +@utils.analyzer("Mypy analyzer", add_mypy_arguments) def main(settings: argparse.Namespace) -> List[utils.ReportData]: cmd = [f"python{settings.version}", '-m', 'mypy'] if settings.config_file: diff --git a/universum/analyzers/pylint.py b/universum/analyzers/pylint.py index 7b383f89..44207403 100644 --- a/universum/analyzers/pylint.py +++ b/universum/analyzers/pylint.py @@ -1,20 +1,17 @@ import argparse import json - from typing import List from . import utils -def pylint_argument_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Pylint analyzer") +def add_pylint_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--rcfile", dest="rcfile", type=str, help="Specify a configuration file.") utils.add_python_version_argument(parser) - return parser @utils.sys_exit -@utils.analyzer(pylint_argument_parser()) +@utils.analyzer("Pylint analyzer", add_pylint_arguments) def main(settings: argparse.Namespace) -> List[utils.ReportData]: cmd = [f"python{settings.version}", '-m', 'pylint', '-f', 'json'] if settings.rcfile: diff --git a/universum/analyzers/uncrustify.py b/universum/analyzers/uncrustify.py index 2f24ca2d..efe0af6c 100755 --- a/universum/analyzers/uncrustify.py +++ b/universum/analyzers/uncrustify.py @@ -1,31 +1,28 @@ import argparse import difflib import os -import shutil import pathlib import re - +import shutil from typing import Callable, List, Optional, Tuple from . import utils -def uncrustify_argument_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Uncrustify analyzer") +def add_uncrustify_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--cfg-file", "-cf", dest="cfg_file", help="Name of the configuration file of Uncrustify; " "can also be set via 'UNCRUSTIFY_CONFIG' env. variable") parser.add_argument("--output-directory", "-od", dest="output_directory", default="uncrustify", help="Directory to store fixed files, generated by Uncrustify " - "and HTML files with diff; the default value is 'uncrustify'" + "and HTML files with diff; the default value is 'uncrustify'. " "Has to be distinct from source directory") parser.add_argument("--report-html", dest="write_html", action="store_true", default=False, help="(optional) Set to generate html reports for each modified file") - return parser @utils.sys_exit -@utils.analyzer(uncrustify_argument_parser()) +@utils.analyzer("Uncrustify analyzer", add_uncrustify_arguments) def main(settings: argparse.Namespace) -> List[utils.ReportData]: if not shutil.which('uncrustify'): raise EnvironmentError("Please install uncrustify") diff --git a/universum/analyzers/utils.py b/universum/analyzers/utils.py index a6bfd452..214c6799 100644 --- a/universum/analyzers/utils.py +++ b/universum/analyzers/utils.py @@ -1,11 +1,12 @@ -import json -import sys import argparse import glob +import json +import os import pathlib import subprocess - +import sys from typing import Any, Callable, List, Optional, Tuple, Set + from typing_extensions import TypedDict from universum.lib.ci_exception import CiException @@ -19,18 +20,30 @@ def __init__(self, code: int = 2, message: Optional[str] = None): self.message: Optional[str] = message -def analyzer(parser: argparse.ArgumentParser): +def create_parser(description: str) -> argparse.ArgumentParser: + module_package = sys.modules["__main__"].__package__ + module_name, _ = os.path.splitext(os.path.basename(sys.argv[0])) + + prog = f"python{sys.version_info.major}.{sys.version_info.minor} -m {module_package}.{module_name}" + return argparse.ArgumentParser(prog=prog, description=description) + + +def analyzer(description: str, add_analyzer_arguments: Callable[[argparse.ArgumentParser], None]): """ Wraps the analyzer specific data and adds common protocol information: --files argument and its processing --result-file argument and its processing This function exists to define analyzer report interface - :param parser: Definition of analyzer custom arguments + :param description: Description of the analyzer, to be used in help + :param add_analyzer_arguments: Function that adds analyzer-specific arguments to parser :return: Wrapped analyzer with common reporting behaviour """ + def internal(func: Callable[[argparse.Namespace], List[ReportData]]) -> Callable[[], List[ReportData]]: def wrapper() -> List[ReportData]: + parser = create_parser(description) + add_analyzer_arguments(parser) add_files_argument(parser) add_result_file_argument(parser) settings: argparse.Namespace = parser.parse_args() @@ -75,6 +88,7 @@ def sys_exit(func: Callable[[], Any]) -> Callable[[], None]: >>> wrap_system_exit(sys_exit(_raise_custom)) 3 """ + def wrapper() -> None: exit_code: int try: From 1dd69773ecf3181f0bfe9f45a3bba64448ab0af1 Mon Sep 17 00:00:00 2001 From: i-keliukh <46344924+i-keliukh@users.noreply.github.com> Date: Fri, 7 Apr 2023 18:02:11 +0300 Subject: [PATCH 25/32] fix(analyzer): documentation generation broken by last change Partially reverted the last change to allow Sphinx argparse extension to work. --- doc/code_report.rst | 3 --- universum/analyzers/mypy.py | 6 ++++-- universum/analyzers/pylint.py | 6 ++++-- universum/analyzers/uncrustify.py | 6 ++++-- universum/analyzers/utils.py | 14 +++++--------- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/doc/code_report.rst b/doc/code_report.rst index a9ca4b86..c16e9ec3 100644 --- a/doc/code_report.rst +++ b/doc/code_report.rst @@ -62,7 +62,6 @@ Pylint .. argparse:: :ref: universum.analyzers.pylint.pylint_argument_parser - :prog: {python} -m universum.analyzers.pylint Config example for ``universum.analyzers.pylint``: @@ -99,7 +98,6 @@ Mypy .. argparse:: :ref: universum.analyzers.mypy.mypy_argument_parser - :prog: {python} -m universum.analyzers.mypy Config example for ``universum.analyzers.mypy``: @@ -136,7 +134,6 @@ Uncrustify .. argparse:: :ref: universum.analyzers.uncrustify.uncrustify_argument_parser - :prog: {python} -m universum.analyzers.uncrustify :nodefault: Config example for ``universum.analyzers.uncrustify``: diff --git a/universum/analyzers/mypy.py b/universum/analyzers/mypy.py index 8536c835..a8b0544a 100644 --- a/universum/analyzers/mypy.py +++ b/universum/analyzers/mypy.py @@ -4,13 +4,15 @@ from . import utils -def add_mypy_arguments(parser: argparse.ArgumentParser) -> None: +def mypy_argument_parser() -> argparse.ArgumentParser: + parser = utils.create_parser("Mypy analyzer", __file__) parser.add_argument("--config-file", dest="config_file", type=str, help="Specify a configuration file.") utils.add_python_version_argument(parser) + return parser @utils.sys_exit -@utils.analyzer("Mypy analyzer", add_mypy_arguments) +@utils.analyzer(mypy_argument_parser()) def main(settings: argparse.Namespace) -> List[utils.ReportData]: cmd = [f"python{settings.version}", '-m', 'mypy'] if settings.config_file: diff --git a/universum/analyzers/pylint.py b/universum/analyzers/pylint.py index 44207403..11cb8a67 100644 --- a/universum/analyzers/pylint.py +++ b/universum/analyzers/pylint.py @@ -5,13 +5,15 @@ from . import utils -def add_pylint_arguments(parser: argparse.ArgumentParser) -> None: +def pylint_argument_parser() -> argparse.ArgumentParser: + parser = utils.create_parser("Pylint analyzer", __file__) parser.add_argument("--rcfile", dest="rcfile", type=str, help="Specify a configuration file.") utils.add_python_version_argument(parser) + return parser @utils.sys_exit -@utils.analyzer("Pylint analyzer", add_pylint_arguments) +@utils.analyzer(pylint_argument_parser()) def main(settings: argparse.Namespace) -> List[utils.ReportData]: cmd = [f"python{settings.version}", '-m', 'pylint', '-f', 'json'] if settings.rcfile: diff --git a/universum/analyzers/uncrustify.py b/universum/analyzers/uncrustify.py index efe0af6c..cfcc41df 100755 --- a/universum/analyzers/uncrustify.py +++ b/universum/analyzers/uncrustify.py @@ -9,7 +9,8 @@ from . import utils -def add_uncrustify_arguments(parser: argparse.ArgumentParser) -> None: +def uncrustify_argument_parser() -> argparse.ArgumentParser: + parser = utils.create_parser("Uncrustify analyzer", __file__) parser.add_argument("--cfg-file", "-cf", dest="cfg_file", help="Name of the configuration file of Uncrustify; " "can also be set via 'UNCRUSTIFY_CONFIG' env. variable") @@ -19,10 +20,11 @@ def add_uncrustify_arguments(parser: argparse.ArgumentParser) -> None: "Has to be distinct from source directory") parser.add_argument("--report-html", dest="write_html", action="store_true", default=False, help="(optional) Set to generate html reports for each modified file") + return parser @utils.sys_exit -@utils.analyzer("Uncrustify analyzer", add_uncrustify_arguments) +@utils.analyzer(uncrustify_argument_parser()) def main(settings: argparse.Namespace) -> List[utils.ReportData]: if not shutil.which('uncrustify'): raise EnvironmentError("Please install uncrustify") diff --git a/universum/analyzers/utils.py b/universum/analyzers/utils.py index 214c6799..f754c42c 100644 --- a/universum/analyzers/utils.py +++ b/universum/analyzers/utils.py @@ -20,30 +20,26 @@ def __init__(self, code: int = 2, message: Optional[str] = None): self.message: Optional[str] = message -def create_parser(description: str) -> argparse.ArgumentParser: - module_package = sys.modules["__main__"].__package__ - module_name, _ = os.path.splitext(os.path.basename(sys.argv[0])) +def create_parser(description: str, module_path: str) -> argparse.ArgumentParser: + module_name, _ = os.path.splitext(os.path.basename(module_path)) - prog = f"python{sys.version_info.major}.{sys.version_info.minor} -m {module_package}.{module_name}" + prog = f"python{sys.version_info.major}.{sys.version_info.minor} -m {__package__}.{module_name}" return argparse.ArgumentParser(prog=prog, description=description) -def analyzer(description: str, add_analyzer_arguments: Callable[[argparse.ArgumentParser], None]): +def analyzer(parser: argparse.ArgumentParser): """ Wraps the analyzer specific data and adds common protocol information: --files argument and its processing --result-file argument and its processing This function exists to define analyzer report interface - :param description: Description of the analyzer, to be used in help - :param add_analyzer_arguments: Function that adds analyzer-specific arguments to parser + :param parser: Definition of analyzer custom arguments :return: Wrapped analyzer with common reporting behaviour """ def internal(func: Callable[[argparse.Namespace], List[ReportData]]) -> Callable[[], List[ReportData]]: def wrapper() -> List[ReportData]: - parser = create_parser(description) - add_analyzer_arguments(parser) add_files_argument(parser) add_result_file_argument(parser) settings: argparse.Namespace = parser.parse_args() From d06d6087ce12dd58f967dcd6d9d562b8499fb617 Mon Sep 17 00:00:00 2001 From: Oleksandr Andrieiev <40266373+o-andrieiev@users.noreply.github.com> Date: Mon, 10 Apr 2023 17:56:52 +0300 Subject: [PATCH 26/32] fix(configuration): allow False value for Step Replaced test of truthiness with an explicit check of None value Resolves #764 Tested by doctests --- universum/configuration_support.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/universum/configuration_support.py b/universum/configuration_support.py index f5706e06..75fafe40 100644 --- a/universum/configuration_support.py +++ b/universum/configuration_support.py @@ -344,7 +344,7 @@ def get(self, key: str, default: Any = None) -> Any: ... warnings.simplefilter("always") ... f() ... return w - >>> step = Step(name='foo', my_var='bar') + >>> step = Step(name='foo', my_var='bar', t1=None, t2=False) >>> do_and_get_warnings(lambda : step.get('name', 'test')) # doctest: +ELLIPSIS [] @@ -356,12 +356,16 @@ def get(self, key: str, default: Any = None) -> Any: 'test' >>> step.get('command', 'test') 'test' + >>> step.get('t1') is None + True + >>> step.get('t2') + False """ result = self._extras.get(key) - if result: + if result is not None: # for custom fields there is a distinction between None and falsy values return result result = self.__dict__.get(key) - if result: + if result: # non-custom fields initialized with falsy values warn("Using legacy API to access configuration values. Please use var." + key + " instead.") return result return default From e4db39b15cfe3a330ba915fa925e5e0c5379c0df Mon Sep 17 00:00:00 2001 From: Oleksandr Andrieiev <40266373+o-andrieiev@users.noreply.github.com> Date: Tue, 11 Apr 2023 11:47:30 +0300 Subject: [PATCH 27/32] fix(report): remove redundant file Static_analysis_report.json is not used for core report functionality, but is added to artifacts and creates misleading assumption that it has some deeper meaning. --- universum/modules/code_report_collector.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/universum/modules/code_report_collector.py b/universum/modules/code_report_collector.py index e558814c..edb2e795 100644 --- a/universum/modules/code_report_collector.py +++ b/universum/modules/code_report_collector.py @@ -92,9 +92,6 @@ def report_code_report_results(self) -> None: if text: report = json.loads(text) - json_file: TextIO = self.artifacts.create_text_file("Static_analysis_report.json") - json_file.write(json.dumps(report, indent=4)) - issue_count: int if not report and report != []: self.out.log_error("There are no results in code report file. Something went wrong.") From edd9c0c0c28d3ea4979cfc4b55cebc45856fe6d4 Mon Sep 17 00:00:00 2001 From: i-keliukh <46344924+i-keliukh@users.noreply.github.com> Date: Wed, 12 Apr 2023 10:35:47 +0300 Subject: [PATCH 28/32] feat(analyzer): code report based on clang-format Common code for clang_format and uncrustify analyzers is placed to utils. Also fixed some error messages of analyzers, fixed a test failure due to another docker engine update and added doc string for one unclear test. --- doc/code_report.rst | 39 +++++++- pylintrc | 5 +- setup.py | 6 +- tests/test_code_report.py | 63 +++++++++---- tests/test_nonci.py | 2 +- universum/analyzers/clang_format.py | 67 ++++++++++++++ universum/analyzers/diff_utils.py | 133 ++++++++++++++++++++++++++++ universum/analyzers/uncrustify.py | 131 +++------------------------ universum/analyzers/utils.py | 14 ++- 9 files changed, 318 insertions(+), 142 deletions(-) create mode 100755 universum/analyzers/clang_format.py create mode 100644 universum/analyzers/diff_utils.py diff --git a/doc/code_report.rst b/doc/code_report.rst index c16e9ec3..ecf06fcd 100644 --- a/doc/code_report.rst +++ b/doc/code_report.rst @@ -6,6 +6,7 @@ The following analysing modules (analysers) are currently added to Universum: * `pylint`_ * `mypy`_ * `uncrustify`_ + * `clang-format`_ Analysers are separate scripts, fully compatible with Universum. It is possible to use them as independent Python modules. @@ -143,7 +144,7 @@ Config example for ``universum.analyzers.uncrustify``: from universum.configuration_support import Configuration, Step configs = Configuration([Step(name="uncrustify", code_report=True, command=[ - "{python}", "-m", "universum.analyzers.uncrustify", "--files", "/home/user/workspace/temp", + "{python}", "-m", "universum.analyzers.uncrustify", "--files", "/home/user/workspace/temp/*.c", "--cfg-file", "file_name.cfg", "--result-file", "${CODE_REPORT_FILE}", "--output-directory", "uncrustify" ])]) @@ -161,4 +162,38 @@ will produce this list of configurations: .. testoutput:: $ ./.universum.py - [{'name': 'uncrustify', 'code_report': True, 'command': '{python} -m universum.analyzers.uncrustify --files /home/user/workspace/temp --cfg-file file_name.cfg --result-file ${CODE_REPORT_FILE} --output-directory uncrustify'}] + [{'name': 'uncrustify', 'code_report': True, 'command': '{python} -m universum.analyzers.uncrustify --files /home/user/workspace/temp/*.c --cfg-file file_name.cfg --result-file ${CODE_REPORT_FILE} --output-directory uncrustify'}] + +Clang-format +------------ + +.. argparse:: + :ref: universum.analyzers.clang-format.clang_format_argument_parser + :nodefault: + +Config example for ``universum.analyzers.clang-format``: + +.. testcode:: + + from universum.configuration_support import Configuration, Step + + configs = Configuration([Step(name="clang-format", code_report=True, command=[ + "{python}", "-m", "universum.analyzers.clang-format", "--files", "/home/user/workspace/temp/*.c", + "--result-file", "${CODE_REPORT_FILE}", "--output-directory", "clang-format", "-e", "clang-format-15", + ])]) + + if __name__ == '__main__': + print(configs.dump()) + +will produce this list of configurations: + +.. testcode:: + :hide: + + print("$ ./.universum.py") + print(configs.dump()) + +.. testoutput:: + + $ ./.universum.py + [{'name': 'clang-format', 'code_report': True, 'command': '{python} -m universum.analyzers.clang-format --files /home/user/workspace/temp/*.c --result-file ${CODE_REPORT_FILE} --output-directory clang-format -e clang-format-15'}] diff --git a/pylintrc b/pylintrc index 7b39d854..ee326fca 100644 --- a/pylintrc +++ b/pylintrc @@ -37,4 +37,7 @@ max-parents = 10 max-line-length = 130 [SIMILARITIES] -ignore-imports=yes \ No newline at end of file +ignore-imports=yes + +[TYPECHECK] +signature-mutators=universum.analyzers.utils.analyzer diff --git a/setup.py b/setup.py index 642aa26c..da23d2c1 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,8 @@ def readme(): 'sh', 'lxml', 'typing-extensions', - 'ansi2html' + 'ansi2html', + 'pyyaml==6.0' ], extras_require={ 'p4': [p4], @@ -56,7 +57,8 @@ def readme(): 'coverage', 'mypy', 'types-requests', - 'selenium==3.141' + 'selenium==3.141', + 'types-PyYAML==6.0' ] }, package_data={'': ['*.css', '*.js']} diff --git a/tests/test_code_report.py b/tests/test_code_report.py index 9e57306c..f73cd2db 100644 --- a/tests/test_code_report.py +++ b/tests/test_code_report.py @@ -16,7 +16,7 @@ def fixture_runner_with_analyzers(docker_main: UniversumRunner): docker_main.environment.install_python_module("pylint") docker_main.environment.install_python_module("mypy") - docker_main.environment.assert_successful_execution("apt install uncrustify") + docker_main.environment.assert_successful_execution("apt install -y uncrustify clang-format") yield docker_main @@ -128,6 +128,11 @@ def finalize(self) -> str: input_tab_size = 2 """ +config_clang_format = """ +--- +AllowShortFunctionsOnASingleLine: Empty +""" + log_fail = r'Found [0-9]+ issues' log_success = r'Issues not found' @@ -167,7 +172,10 @@ def test_code_report_direct_log(runner_with_analyzers: UniversumRunner, tested_c @pytest.mark.parametrize('analyzers, extra_args, tested_content, expected_success', [ [['uncrustify'], [], source_code_c, True], - [['uncrustify'], [], source_code_c.replace('\t', ' '), False], + [['uncrustify'], [], source_code_c.replace('\t', ' '), False], # by default uncrustify converts spaces to tabs + [['clang_format'], [], source_code_c.replace('\t', ' '), True], # by default clang-format expands tabs to 2 spaces + [['clang_format'], [], source_code_c.replace('\t', ' '), False], + [['clang_format', 'uncrustify'], [], source_code_c.replace('\t', ' '), False], [['pylint', 'mypy'], ["--python-version", python_version()], source_code_python, True], [['pylint'], ["--python-version", python_version()], source_code_python + '\n', False], [['mypy'], ["--python-version", python_version()], source_code_python.replace(': str', ': int'), False], @@ -177,6 +185,9 @@ def test_code_report_direct_log(runner_with_analyzers: UniversumRunner, tested_c ], ids=[ 'uncrustify_no_issues', 'uncrustify_found_issues', + 'clang_format_no_issues', + 'clang_format_found_issues', + 'clang_format_and_uncrustify_found_issues', 'pylint_and_mypy_both_no_issues', 'pylint_found_issues', 'mypy_found_issues', @@ -194,6 +205,9 @@ def test_code_report_log(runner_with_analyzers: UniversumRunner, analyzers, extr if analyzer == 'uncrustify': args += ["--cfg-file", "cfg"] (runner_with_analyzers.local.root_directory / "cfg").write_text(config_uncrustify) + elif analyzer == 'clang_format': + (runner_with_analyzers.local.root_directory / ".clang-format").write_text(config_clang_format) + config.add_analyzer(analyzer, args) log = runner_with_analyzers.run(config.finalize()) @@ -210,7 +224,7 @@ def test_without_code_report_command(runner_with_analyzers: UniversumRunner): assert not pattern.findall(log) -@pytest.mark.parametrize('analyzer', ['pylint', 'mypy', 'uncrustify']) +@pytest.mark.parametrize('analyzer', ['pylint', 'mypy', 'uncrustify', 'clang_format']) @pytest.mark.parametrize('arg_set, expected_log', [ [["--files", "source_file.py"], "error: the following arguments are required: --result-file"], [["--files", "source_file.py", "--result-file"], "result-file: expected one argument"], @@ -235,10 +249,12 @@ def test_analyzer_python_version_params(runner_with_analyzers: UniversumRunner, "--result-file", "${CODE_REPORT_FILE}", '--rcfile'], "rcfile: expected one argument"], ['uncrustify', ["--files", "source_file", "--result-file", "${CODE_REPORT_FILE}"], - "Please specify the '--cfg_file' parameter or set an env. variable 'UNCRUSTIFY_CONFIG'"], + "Please specify the '--cfg-file' parameter or set 'UNCRUSTIFY_CONFIG' environment variable"], ['uncrustify', ["--files", "source_file", "--result-file", "${CODE_REPORT_FILE}", "--cfg-file", "cfg", "--output-directory", "."], - "Target and source folders for uncrustify are not allowed to match"], + "Target folder must not be identical to source folder"], + ['clang_format', ["--files", "source_file", "--result-file", "${CODE_REPORT_FILE}", "--output-directory", "."], + "Target folder must not be identical to source folder"], ]) def test_analyzer_specific_params(runner_with_analyzers: UniversumRunner, analyzer, arg_set, expected_log): source_file = runner_with_analyzers.local.root_directory / "source_file" @@ -249,39 +265,54 @@ def test_analyzer_specific_params(runner_with_analyzers: UniversumRunner, analyz assert expected_log in log, f"'{expected_log}' is not found in '{log}'" -@pytest.mark.parametrize('extra_args, tested_content, expected_success, expected_artifact', [ - [[], source_code_c, True, False], - [["--report-html"], source_code_c.replace('\t', ' '), False, True], - [[], source_code_c.replace('\t', ' '), False, False], +@pytest.mark.parametrize('analyzer, extra_args, tested_content, expected_success, expected_artifact', [ + ['uncrustify', ["--report-html"], source_code_c, True, False], + ['uncrustify', ["--report-html"], source_code_c.replace('\t', ' '), False, True], + ['uncrustify', [], source_code_c.replace('\t', ' '), False, False], + ['clang_format', ["--report-html"], source_code_c.replace('\t', ' '), True, False], + ['clang_format', ["--report-html"], source_code_c, False, True], + ['clang_format', [], source_code_c, False, False], ], ids=[ "uncrustify_html_file_not_needed", "uncrustify_html_file_saved", "uncrustify_html_file_disabled", + "clang_format_html_file_not_needed", + "clang_format_html_file_saved", + "clang_format_html_file_disabled", ]) -def test_uncrustify_file_diff(runner_with_analyzers: UniversumRunner, - extra_args, tested_content, expected_success, expected_artifact): +def test_diff_html_file(runner_with_analyzers: UniversumRunner, analyzer, + extra_args, tested_content, expected_success, expected_artifact): + root = runner_with_analyzers.local.root_directory source_file = root / "source_file" source_file.write_text(tested_content) - (root / "cfg").write_text(config_uncrustify) common_args = [ "--result-file", "${CODE_REPORT_FILE}", "--files", "source_file", - "--cfg-file", "cfg", + "--output-directory", "diff_temp" ] + if analyzer == 'uncrustify': + (root / "cfg").write_text(config_uncrustify) + common_args.extend(["--cfg-file", "cfg"]) + elif analyzer == 'clang_format': + (root / ".clang-format").write_text(config_clang_format) + args = common_args + extra_args - extra_config = "artifacts='./uncrustify/source_file.html'" - log = runner_with_analyzers.run(ConfigData().add_analyzer('uncrustify', args, extra_config).finalize()) + extra_config = "artifacts='./diff_temp/source_file.html'" + log = runner_with_analyzers.run(ConfigData().add_analyzer(analyzer, args, extra_config).finalize()) expected_log = log_success if expected_success else log_fail assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" expected_artifacts_state = "Success" if expected_artifact else "Failed" - expected_log = f"Collecting artifacts for the 'Run uncrustify' step - [^\n]*{expected_artifacts_state}" + expected_log = f"Collecting artifacts for the 'Run {analyzer}' step - [^\n]*{expected_artifacts_state}" assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" def test_code_report_extended_arg_search(tmp_path: pathlib.Path, stdout_checker: FuzzyCallChecker): + """ + Test if ${CODE_REPORT_FILE} is replaced not only in --result-file argument of the Step + """ env = utils.LocalTestEnvironment(tmp_path, "main") env.settings.Vcs.type = "none" env.settings.LocalMainVcs.source_dir = str(tmp_path) diff --git a/tests/test_nonci.py b/tests/test_nonci.py index fe24168d..d03bb4f0 100644 --- a/tests/test_nonci.py +++ b/tests/test_nonci.py @@ -23,7 +23,7 @@ def test_launcher_output(docker_nonci: UniversumRunner): - version control and review system are not used - project root is set to current directory """ - cwd = str(docker_nonci.local.root_directory) + cwd = str(docker_nonci.local.root_directory.absolute()) artifacts = docker_nonci.artifact_dir file_output_expected = f"Adding file {artifacts}/test_step_log.txt to artifacts" pwd_string_in_logs = f"pwd:[{cwd}]" diff --git a/universum/analyzers/clang_format.py b/universum/analyzers/clang_format.py new file mode 100755 index 00000000..56a64f80 --- /dev/null +++ b/universum/analyzers/clang_format.py @@ -0,0 +1,67 @@ +import argparse +import pathlib +from typing import Callable, List, Optional, Tuple + +import yaml + +from . import utils, diff_utils + + +def clang_format_argument_parser() -> argparse.ArgumentParser: + parser = diff_utils.diff_analyzer_argument_parser("Clang-format analyzer", __file__, "clang-format") + parser.add_argument("--executable", "-e", dest="executable", default="clang-format", + help="The name of the clang-format executable, default: clang-format") + parser.add_argument("--style", dest="style", + help="The 'style' parameter of the clang-format. Can be literal 'file' string or " + "path to real file. See the clang-format documentation for details.") + return parser + + +def _add_style_param_if_present(cmd: List[str], settings: argparse.Namespace) -> None: + if settings.style: + cmd.extend(["-style", settings.style]) + + +@utils.sys_exit +@utils.analyzer(clang_format_argument_parser()) +def main(settings: argparse.Namespace) -> List[utils.ReportData]: + settings.name = "clang-format" + diff_utils.diff_analyzer_common_main(settings) + + html_diff_file_writer: Optional[Callable[[pathlib.Path, List[str], List[str]], None]] = None + if settings.write_html: + wrapcolumn, tabsize = _get_wrapcolumn_tabsize(settings) + html_diff_file_writer = diff_utils.HtmlDiffFileWriter(settings.target_folder, wrapcolumn, tabsize) + + files: List[Tuple[pathlib.Path, pathlib.Path]] = [] + for src_file_absolute, target_file_absolute, _ in utils.get_files_with_absolute_paths(settings): + files.append((src_file_absolute, target_file_absolute)) + cmd = [settings.executable, src_file_absolute] + _add_style_param_if_present(cmd, settings) + output, _ = utils.run_for_output(cmd) + with open(target_file_absolute, "w", encoding="utf-8") as output_file: + output_file.write(output) + + return diff_utils.diff_analyzer_output_parser(files, html_diff_file_writer) + + +def _get_wrapcolumn_tabsize(settings: argparse.Namespace) -> Tuple[int, int]: + cmd = [settings.executable, "--dump-config"] + _add_style_param_if_present(cmd, settings) + output, error = utils.run_for_output(cmd) + if error: + raise utils.AnalyzerException(message="clang-format --dump-config failed with the following error output: " + + error) + try: + clang_style = yaml.safe_load(output) + return clang_style["ColumnLimit"], clang_style["IndentWidth"] + except yaml.YAMLError as parse_error: + raise utils.AnalyzerException(message="Parsing of clang-format config produced the following error: " + + str(parse_error)) + except KeyError as key_error: + raise utils.AnalyzerException(message="Cannot find key in yaml during parsing of clang-format config: " + + str(key_error)) + + +if __name__ == "__main__": + main() diff --git a/universum/analyzers/diff_utils.py b/universum/analyzers/diff_utils.py new file mode 100644 index 00000000..493cf7de --- /dev/null +++ b/universum/analyzers/diff_utils.py @@ -0,0 +1,133 @@ +import argparse +import difflib +import pathlib +import shutil +from typing import Callable, List, Optional, Tuple + +from . import utils + + +class HtmlDiffFileWriter: + + def __init__(self, target_folder: pathlib.Path, wrapcolumn: int, tabsize: int) -> None: + self.target_folder = target_folder + self.differ = difflib.HtmlDiff(tabsize=tabsize, wrapcolumn=wrapcolumn) + + def __call__(self, file: pathlib.Path, src: List[str], target: List[str]) -> None: + file_relative = file.relative_to(pathlib.Path.cwd()) + out_file_name: str = str(file_relative).replace('/', '_') + '.html' + with open(self.target_folder.joinpath(out_file_name), 'w', encoding="utf-8") as out_file: + out_file.write(self.differ.make_file(src, target, context=False)) + + +DiffWriter = Callable[[pathlib.Path, List[str], List[str]], None] + + +def diff_analyzer_output_parser(files: List[Tuple[pathlib.Path, pathlib.Path]], + write_diff_file: Optional[DiffWriter] + ) -> List[utils.ReportData]: + result: List[utils.ReportData] = [] + for src_file, dst_file in files: + with open(src_file, encoding="utf-8") as src: + src_lines = src.readlines() + with open(dst_file, encoding="utf-8") as fixed: + fixed_lines = fixed.readlines() + + issues = _get_issues_from_diff(src_file, src_lines, fixed_lines) + if issues and write_diff_file: + write_diff_file(src_file, src_lines, fixed_lines) + result.extend(issues) + return result + + +def _get_issues_from_diff(src_file: pathlib.Path, src: List[str], target: List[str]) -> List[utils.ReportData]: + result = [] + matching_blocks: List[difflib.Match] = \ + difflib.SequenceMatcher(a=src, b=target).get_matching_blocks() + previous_match = matching_blocks[0] + for match in matching_blocks[1:]: + block = _get_mismatching_block(previous_match, match, src, target) + previous_match = match + if not block: + continue + line, before, after = block + path: pathlib.Path = src_file.relative_to(pathlib.Path.cwd()) + message = _get_issue_message(before, after) + result.append(utils.ReportData( + symbol="Code Style issue", + message=message, + path=str(path), + line=line + )) + + return result + + +def _get_issue_message(before: str, after: str) -> str: + # The maximum number of lines to write separate comments for + # If exceeded, summarized comment will be provided instead + max_lines = 11 + diff_size = len(before.splitlines()) + if diff_size > max_lines: + message = f"\nLarge block of code ({diff_size} lines) has issues\n" + \ + f"Non-compliant code blocks exceeding {max_lines} lines are not reported\n" + else: + # Message with before&after + message = f"\nOriginal code:\n```diff\n{before}```\n" + \ + f"Fixed code:\n```diff\n{after}```\n" + return message + + +def _get_mismatching_block(previous_match: difflib.Match, # src[a:a+size] = target[b:b+size] + match: difflib.Match, + src: List[str], target: List[str] + ) -> Optional[Tuple[int, str, str]]: + previous_match_end_in_src = previous_match.a + previous_match.size + previous_match_end_in_target = previous_match.b + previous_match.size + match_start_in_src = match.a + match_start_in_target = match.b + if previous_match_end_in_src == match_start_in_src: + return None + line = match_start_in_src + before = _get_text_for_block(previous_match_end_in_src - 1, match_start_in_src, src) + after = _get_text_for_block(previous_match_end_in_target - 1, match_start_in_target, target) + return line, before, after + + +def _get_text_for_block(start: int, end: int, lines: List[str]) -> str: + return _replace_whitespace_characters(''.join(lines[start: end])) + + +_whitespace_character_mapping = { + " ": "\u00b7", + "\t": "\u2192\u2192\u2192\u2192", + "\n": "\u2193\u000a" +}.items() + + +def _replace_whitespace_characters(line: str) -> str: + for old_str, new_str in _whitespace_character_mapping: + line = line.replace(old_str, new_str) + return line + + +def diff_analyzer_argument_parser(description: str, module_path: str, output_directory: str) -> argparse.ArgumentParser: + parser = utils.create_parser(description, module_path) + parser.add_argument("--output-directory", "-od", dest="output_directory", default=output_directory, + help=f"Directory to store fixed files and HTML files with diff; the default " + f"value is '{output_directory}'. Has to be distinct from source directory") + parser.add_argument("--report-html", dest="write_html", action="store_true", default=False, + help="(optional) Set to generate html reports for each modified file") + return parser + + +def diff_analyzer_common_main(settings: argparse.Namespace) -> None: + settings.target_folder = utils.normalize_path(settings.output_directory) + if settings.target_folder.exists() and settings.target_folder.samefile(pathlib.Path.cwd()): + raise EnvironmentError("Target folder must not be identical to source folder") + + settings.target_folder.mkdir(parents=True, exist_ok=True) + + if not shutil.which(settings.executable): + raise EnvironmentError(f"{settings.name} executable '{settings.executable}' is not found. " + f"Please install {settings.name} or fix the executable name.") diff --git a/universum/analyzers/uncrustify.py b/universum/analyzers/uncrustify.py index cfcc41df..7afc548d 100755 --- a/universum/analyzers/uncrustify.py +++ b/universum/analyzers/uncrustify.py @@ -1,85 +1,44 @@ import argparse -import difflib import os import pathlib import re -import shutil from typing import Callable, List, Optional, Tuple -from . import utils +from . import utils, diff_utils def uncrustify_argument_parser() -> argparse.ArgumentParser: - parser = utils.create_parser("Uncrustify analyzer", __file__) + parser = diff_utils.diff_analyzer_argument_parser("Uncrustify analyzer", __file__, "uncrustify") parser.add_argument("--cfg-file", "-cf", dest="cfg_file", help="Name of the configuration file of Uncrustify; " "can also be set via 'UNCRUSTIFY_CONFIG' env. variable") - parser.add_argument("--output-directory", "-od", dest="output_directory", default="uncrustify", - help="Directory to store fixed files, generated by Uncrustify " - "and HTML files with diff; the default value is 'uncrustify'. " - "Has to be distinct from source directory") - parser.add_argument("--report-html", dest="write_html", action="store_true", default=False, - help="(optional) Set to generate html reports for each modified file") return parser @utils.sys_exit @utils.analyzer(uncrustify_argument_parser()) def main(settings: argparse.Namespace) -> List[utils.ReportData]: - if not shutil.which('uncrustify'): - raise EnvironmentError("Please install uncrustify") + settings.name = "uncrustify" + settings.executable = "uncrustify" + diff_utils.diff_analyzer_common_main(settings) + if not settings.cfg_file and 'UNCRUSTIFY_CONFIG' not in os.environ: - raise EnvironmentError("Please specify the '--cfg_file' parameter " - "or set an env. variable 'UNCRUSTIFY_CONFIG'") - target_folder: pathlib.Path = utils.normalize(settings.output_directory) - if target_folder.exists() and target_folder.samefile(pathlib.Path.cwd()): - raise EnvironmentError("Target and source folders for uncrustify are not allowed to match") + raise EnvironmentError("Please specify the '--cfg-file' parameter " + "or set 'UNCRUSTIFY_CONFIG' environment variable") + html_diff_file_writer: Optional[Callable[[pathlib.Path, List[str], List[str]], None]] = None if settings.write_html: wrapcolumn, tabsize = _get_wrapcolumn_tabsize(settings.cfg_file) - html_diff_file_writer = HtmlDiffFileWriter(target_folder, wrapcolumn, tabsize) + html_diff_file_writer = diff_utils.HtmlDiffFileWriter(settings.target_folder, wrapcolumn, tabsize) cmd = ["uncrustify", "-q", "-c", settings.cfg_file, "--prefix", settings.output_directory] files: List[Tuple[pathlib.Path, pathlib.Path]] = [] - for src_file in settings.file_list: - src_file_absolute = utils.normalize(src_file) - src_file_relative = src_file_absolute.relative_to(pathlib.Path.cwd()) - target_file_absolute: pathlib.Path = target_folder.joinpath(src_file_relative) + for src_file_absolute, target_file_absolute, src_file_relative in utils.get_files_with_absolute_paths(settings): files.append((src_file_absolute, target_file_absolute)) cmd.append(src_file_relative) utils.run_for_output(cmd) - return uncrustify_output_parser(files, html_diff_file_writer) - - -class HtmlDiffFileWriter: - - def __init__(self, target_folder: pathlib.Path, wrapcolumn: int, tabsize: int) -> None: - self.target_folder = target_folder - self.differ = difflib.HtmlDiff(tabsize=tabsize, wrapcolumn=wrapcolumn) - - def __call__(self, file: pathlib.Path, src: List[str], target: List[str]) -> None: - file_relative = file.relative_to(pathlib.Path.cwd()) - out_file_name: str = str(file_relative).replace('/', '_') + '.html' - with open(self.target_folder.joinpath(out_file_name), 'w', encoding="utf-8") as out_file: - out_file.write(self.differ.make_file(src, target, context=False)) - - -def uncrustify_output_parser(files: List[Tuple[pathlib.Path, pathlib.Path]], - write_diff_file: Optional[Callable[[pathlib.Path, List[str], List[str]], None]] - ) -> List[utils.ReportData]: - result: List[utils.ReportData] = [] - for src_file, uncrustify_file in files: - with open(src_file, encoding="utf-8") as src: - src_lines = src.readlines() - with open(uncrustify_file, encoding="utf-8") as fixed: - fixed_lines = fixed.readlines() - - issues = _get_issues_from_diff(src_file, src_lines, fixed_lines) - if issues and write_diff_file: - write_diff_file(src_file, src_lines, fixed_lines) - result.extend(issues) - return result + return diff_utils.diff_analyzer_output_parser(files, html_diff_file_writer) def _get_wrapcolumn_tabsize(cfg_file: str) -> Tuple[int, int]: @@ -98,69 +57,5 @@ def _get_wrapcolumn_tabsize(cfg_file: str) -> Tuple[int, int]: return wrapcolumn, tabsize -def _get_issues_from_diff(src_file: pathlib.Path, src: List[str], target: List[str]) -> List[utils.ReportData]: - result = [] - matching_blocks: List[difflib.Match] = \ - difflib.SequenceMatcher(a=src, b=target).get_matching_blocks() - previous_match = matching_blocks[0] - for match in matching_blocks[1:]: - block = _get_mismatching_block(previous_match, match, src, target) - previous_match = match - if not block: - continue - line, before, after = block - path: pathlib.Path = src_file.relative_to(pathlib.Path.cwd()) - message = _get_issue_message(before, after) - result.append(utils.ReportData( - symbol="Uncrustify Code Style issue", - message=message, - path=str(path), - line=line - )) - - return result - - -def _get_issue_message(before: str, after: str) -> str: - # The maximum number of lines to write separate comments for - # If exceeded, summarized comment will be provided instead - max_lines = 11 - diff_size = len(before.splitlines()) - if diff_size > max_lines: - message = f"\nLarge block of code ({diff_size} lines) has issues\n" + \ - f"Non-compliant code blocks exceeding {max_lines} lines are not reported\n" - else: - # Message with before&after - message = f"\nOriginal code:\n```diff\n{before}```\n" + \ - f"Uncrustify generated code:\n```diff\n{after}```\n" - return message - - -def _get_mismatching_block(previous_match: difflib.Match, # src[a:a+size] = target[b:b+size] - match: difflib.Match, - src: List[str], target: List[str] - ) -> Optional[Tuple[int, str, str]]: - previous_match_end_in_src = previous_match.a + previous_match.size - previous_match_end_in_target = previous_match.b + previous_match.size - match_start_in_src = match.a - match_start_in_target = match.b - if previous_match_end_in_src == match_start_in_src: - return None - line = match_start_in_src - before = _get_text_for_block(previous_match_end_in_src - 1, match_start_in_src, src) - after = _get_text_for_block(previous_match_end_in_target - 1, match_start_in_target, target) - return line, before, after - - -def _get_text_for_block(start: int, end: int, lines: List[str]) -> str: - return _replace_invisible_symbols(''.join(lines[start: end])) - - -def _replace_invisible_symbols(line: str) -> str: - for old_str, new_str in zip([" ", "\t", "\n"], ["\u00b7", "\u2192\u2192\u2192\u2192", "\u2193\u000a"]): - line = line.replace(old_str, new_str) - return line - - if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter # see https://github.com/PyCQA/pylint/issues/259 + main() diff --git a/universum/analyzers/utils.py b/universum/analyzers/utils.py index f754c42c..7d921709 100644 --- a/universum/analyzers/utils.py +++ b/universum/analyzers/utils.py @@ -5,7 +5,7 @@ import pathlib import subprocess import sys -from typing import Any, Callable, List, Optional, Tuple, Set +from typing import Any, Callable, List, Optional, Tuple, Set, Iterable from typing_extensions import TypedDict @@ -156,6 +156,16 @@ def report_to_file(issues: List[ReportData], json_file: Optional[str] = None) -> sys.stdout.write(issues_json) -def normalize(file: str) -> pathlib.Path: +def normalize_path(file: str) -> pathlib.Path: file_path = pathlib.Path(file) return file_path if file_path.is_absolute() else pathlib.Path.cwd().joinpath(file_path) + + +def get_files_with_absolute_paths(settings: argparse.Namespace) -> Iterable[Tuple[pathlib.Path, + pathlib.Path, + pathlib.Path]]: + for src_file in settings.file_list: + src_file_absolute = normalize_path(src_file) + src_file_relative = src_file_absolute.relative_to(pathlib.Path.cwd()) + target_file_absolute: pathlib.Path = settings.target_folder.joinpath(src_file_relative) + yield src_file_absolute, target_file_absolute, src_file_relative From 1086bd8e69d7a77de0c533160901e054ffe971c6 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Wed, 12 Apr 2023 19:14:25 +0300 Subject: [PATCH 29/32] fix(workflow): disable link previews in TG bot All previews refer to Universum main page anyway. --- .github/workflows/telegram-bot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/telegram-bot.yml b/.github/workflows/telegram-bot.yml index 6d623abc..1fe265a6 100644 --- a/.github/workflows/telegram-bot.yml +++ b/.github/workflows/telegram-bot.yml @@ -63,7 +63,7 @@ jobs: fi if [[ ! -z $TEXT ]]; then - curl --get --data-urlencode "chat_id=${{ secrets.TELEGRAM_CHAT_ID }}" \ + curl --get --data-urlencode "chat_id=${{ secrets.TELEGRAM_CHAT_ID }}" --data-urlencode "disable_web_page_preview=True" \ --data-urlencode "text=$TEXT" --data-urlencode "parse_mode=HTML" $URL fi From b902d9d0dcd087908481a037cb35730f5c28c35c Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Mon, 24 Apr 2023 12:54:13 +0300 Subject: [PATCH 30/32] fix(workflow): add comment body for review comment --- .github/workflows/telegram-bot.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/telegram-bot.yml b/.github/workflows/telegram-bot.yml index 1fe265a6..c319b739 100644 --- a/.github/workflows/telegram-bot.yml +++ b/.github/workflows/telegram-bot.yml @@ -23,6 +23,7 @@ jobs: PR_MERGED: ${{ github.event.pull_request.merged_by.login }} REVIEW_STATE: ${{ github.event.review.state }} REVIEW_AUTHOR: ${{ github.event.review.user.login }} + REVIEW_COMMENT: ${{ github.event.review.body }} COMMENT_AUTHOR: ${{ github.event.comment.user.login }} COMMENT_URL: ${{ github.event.comment.html_url }} COMMENT_BODY: ${{ github.event.comment.body }} @@ -51,7 +52,10 @@ jobs: elif [[ ! -z "${{ github.event.review }}" && "${{ env.REVIEW_STATE }}" == "changes_requested" ]]; then TEXT=`echo -e "${{ env.REVIEW_AUTHOR }} requested changes for PR#${{ env.PR_NUMBER }}"` - elif [[ ! -z "${{ github.event.review }}" && "${{ env.REVIEW_STATE }}" != "changes_requested" ]]; then + elif [[ ! -z "${{ github.event.review }}" && "${{ env.REVIEW_STATE }}" == "commented" ]]; then + ESCAPED_TEXT=`echo -e "${{ env.REVIEW_COMMENT }}"| sed 's/\&/\&/g' | sed 's//\>/g'` + TEXT=`echo -e "${{ env.REVIEW_AUTHOR }} posted the following comment to PR#${{ env.PR_NUMBER }}:\n$ESCAPED_TEXT"` + elif [[ ! -z "${{ github.event.review }}" ]]; then TEXT=`echo -e "${{ env.REVIEW_AUTHOR }} ${{ env.REVIEW_STATE }} PR#${{ env.PR_NUMBER }}"` elif [[ -z "${{ github.event.review }}" && "${{ github.event.action }}" == "submitted" ]]; then TEXT=`echo -e "Due to GitHub Actions bug we cannot identify, who approved PR#${{ env.PR_NUMBER }}"` From a3b80a5db4db2664edf0311870370988a84d2e41 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Wed, 10 May 2023 12:55:00 +0300 Subject: [PATCH 31/32] fix(swarm): remove warning suppression that is no longer relevant --- universum/modules/vcs/swarm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/universum/modules/vcs/swarm.py b/universum/modules/vcs/swarm.py index 53eb8e46..ca671301 100644 --- a/universum/modules/vcs/swarm.py +++ b/universum/modules/vcs/swarm.py @@ -9,7 +9,7 @@ from ...lib.ci_exception import CiException from ...lib.gravity import Dependency -urllib3.disable_warnings((urllib3.exceptions.InsecurePlatformWarning, urllib3.exceptions.SNIMissingWarning)) # type: ignore +urllib3.disable_warnings(urllib3.exceptions.InsecurePlatformWarning) # type: ignore __all__ = [ "Swarm" From 9f0894a20cc8409a145465d8dba5d1f0520ce4cc Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Wed, 10 May 2023 17:54:22 +0300 Subject: [PATCH 32/32] update(docs): update changelog to Universum 0.19.15 --- CHANGELOG.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0ea21a2d..0678da9b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,23 @@ Change log ========== +0.19.15 (2023-05-10) +-------------------- + +New features +~~~~~~~~~~~~ + +* **analyzer:** code report based on clang-format + +Bug fixes +~~~~~~~~~ + +* **config:** fixed returning "None" instead of "False" for Step custom keys set to "False" +* **artifact:** remove redundant Static_analysis_report.json from artifacts +* **analyzer:** incorrect program name in help +* **report:** set exit code even if reporting crashes + + 0.19.14 (2022-11-14) --------------------