diff --git a/.github/workflows/precommit-check.yml b/.github/workflows/precommit-check.yml new file mode 100644 index 00000000..1700d7eb --- /dev/null +++ b/.github/workflows/precommit-check.yml @@ -0,0 +1,39 @@ +name: Universum check +on: push + +jobs: + universum_check: + name: Universum check + runs-on: ubuntu-latest + + steps: + - name: Setup python 3.8 + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Install dependency + run: pip install universum[test] + + - name: Universum + run: + python -u -m universum + --fail-unsuccessful + --vcs-type=git + --git-repo $GITHUB_SERVER_URL/$GITHUB_REPOSITORY + --git-refspec $GITHUB_REF_NAME + --no-archive + --no-diff + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v1 + if: always() + with: + files: artifacts/*.xml + + - name: Collect artifacts + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: artifacts + path: artifacts diff --git a/.github/workflows/telegram-bot.yml b/.github/workflows/telegram-bot.yml new file mode 100644 index 00000000..c319b739 --- /dev/null +++ b/.github/workflows/telegram-bot.yml @@ -0,0 +1,75 @@ +name: Telegram bot +on: + pull_request: + types: [opened, synchronize, closed] + issue_comment: + types: [created] + workflow_run: + workflows: [Universum check] + types: [completed] + pull_request_review: + types: [submitted, edited, dismissed] + +jobs: + make-comment: + name: Send comment to TG + runs-on: ubuntu-latest + env: + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + PR_NAME: ${{ github.event.pull_request.title }} + PR_BASE: ${{ github.event.pull_request.base.ref }} + PR_URL: ${{ github.event.pull_request.html_url }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_MERGED: ${{ github.event.pull_request.merged_by.login }} + REVIEW_STATE: ${{ github.event.review.state }} + REVIEW_AUTHOR: ${{ github.event.review.user.login }} + REVIEW_COMMENT: ${{ github.event.review.body }} + COMMENT_AUTHOR: ${{ github.event.comment.user.login }} + COMMENT_URL: ${{ github.event.comment.html_url }} + COMMENT_BODY: ${{ github.event.comment.body }} + COMMENT_NUMBER: ${{ github.event.issue.number }} + UNIVERUM_COMMIT: ${{ github.event.workflow_run.head_sha }} + UNIVERSUM_BRANCH: ${{ github.event.workflow_run.head_branch }} + UNIVERSUM_LOG: ${{ github.event.workflow_run.html_url }} + + steps: + - name: Send message to TG + run: | + + if [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "opened" ]]; then + ESCAPED_NAME=`echo -e "${{ env.PR_NAME }}" | sed 's/\&/\&/g' | sed 's//\>/g'` + TEXT=`echo -e "${{ env.PR_AUTHOR }} created new PR#${{ env.PR_NUMBER }} '$ESCAPED_NAME' to branch '${{ env.PR_BASE }}'"` + elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "synchronize" ]]; then + TEXT=`echo -e "${{ env.PR_AUTHOR }} updated PR#${{ env.PR_NUMBER }}"` + elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "closed" && "${{ github.event.pull_request.merged }}" == "true" ]]; then + TEXT=`echo -e "${{ env.PR_MERGED }} merged PR#${{ env.PR_NUMBER }} to branch '${{ env.PR_BASE }}'"` + elif [[ ! -z "${{ github.event.pull_request }}" && "${{ github.event.action }}" == "closed" ]]; then + TEXT=`echo -e "${{ env.PR_AUTHOR }} closed PR#${{ env.PR_NUMBER }}"` + + elif [[ ! -z "${{ github.event.comment }}" ]]; then + ESCAPED_TEXT=`echo -e "${{ env.COMMENT_BODY }}"| sed 's/\&/\&/g' | sed 's//\>/g'` + TEXT=`echo -e "${{ env.COMMENT_AUTHOR }} posted the following comment to issue #${{ env.COMMENT_NUMBER }}:\n$ESCAPED_TEXT"` + + elif [[ ! -z "${{ github.event.review }}" && "${{ env.REVIEW_STATE }}" == "changes_requested" ]]; then + TEXT=`echo -e "${{ env.REVIEW_AUTHOR }} requested changes for PR#${{ env.PR_NUMBER }}"` + elif [[ ! -z "${{ github.event.review }}" && "${{ env.REVIEW_STATE }}" == "commented" ]]; then + ESCAPED_TEXT=`echo -e "${{ env.REVIEW_COMMENT }}"| sed 's/\&/\&/g' | sed 's//\>/g'` + TEXT=`echo -e "${{ env.REVIEW_AUTHOR }} posted the following comment to PR#${{ env.PR_NUMBER }}:\n$ESCAPED_TEXT"` + elif [[ ! -z "${{ github.event.review }}" ]]; then + TEXT=`echo -e "${{ env.REVIEW_AUTHOR }} ${{ env.REVIEW_STATE }} PR#${{ env.PR_NUMBER }}"` + elif [[ -z "${{ github.event.review }}" && "${{ github.event.action }}" == "submitted" ]]; then + TEXT=`echo -e "Due to GitHub Actions bug we cannot identify, who approved PR#${{ env.PR_NUMBER }}"` + + elif [[ ! -z "${{ github.event.workflow_run }}" && "${{ github.event.workflow_run.conclusion }}" == "success" ]]; then + TEXT=`echo -e "Universum run for branch '${{ env.UNIVERSUM_BRANCH }}' SUCCEDED; commit ${{ env.UNIVERUM_COMMIT }} "` + elif [[ ! -z "${{ github.event.workflow_run }}" && "${{ github.event.workflow_run.conclusion }}" == "failure" ]]; then + TEXT=`echo -e "Universum run for branch '${{ env.UNIVERSUM_BRANCH }}' FAILED; commit ${{ env.UNIVERUM_COMMIT }} "` + fi + + if [[ ! -z $TEXT ]]; then + curl --get --data-urlencode "chat_id=${{ secrets.TELEGRAM_CHAT_ID }}" --data-urlencode "disable_web_page_preview=True" \ + --data-urlencode "text=$TEXT" --data-urlencode "parse_mode=HTML" $URL + fi + + env: + URL: https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage diff --git a/.universum.py b/.universum.py index 71dc3b24..e3c3eda8 100644 --- a/.universum.py +++ b/.universum.py @@ -20,12 +20,13 @@ def pip_install(module_name): configs = Variations([Step(name="Create virtual environment", command=[python, "-m", "venv", env_name]), Step(name="Update Docker images", command=run_virtual("make images")), - Step(name="Install Universum for tests", artifacts="junit_results.xml", + Step(name="Install Universum for tests", command=run_virtual(pip_install(".[test]"))), Step(name="Make", artifacts="doc/_build", command=run_virtual("make")), Step(name="Make tests", artifacts="htmlcov", command=run_virtual("export LANG=en_US.UTF-8; make test")), + Step(name="Collect test results", artifacts="junit_results.xml"), Step(name="Run static pylint", code_report=True, command=run_virtual(f"{python} -m universum.analyzers.pylint " diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0ea21a2d..0678da9b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,23 @@ Change log ========== +0.19.15 (2023-05-10) +-------------------- + +New features +~~~~~~~~~~~~ + +* **analyzer:** code report based on clang-format + +Bug fixes +~~~~~~~~~ + +* **config:** fixed returning "None" instead of "False" for Step custom keys set to "False" +* **artifact:** remove redundant Static_analysis_report.json from artifacts +* **analyzer:** incorrect program name in help +* **report:** set exit code even if reporting crashes + + 0.19.14 (2022-11-14) -------------------- diff --git a/doc/code_report.rst b/doc/code_report.rst index a9ca4b86..ecf06fcd 100644 --- a/doc/code_report.rst +++ b/doc/code_report.rst @@ -6,6 +6,7 @@ The following analysing modules (analysers) are currently added to Universum: * `pylint`_ * `mypy`_ * `uncrustify`_ + * `clang-format`_ Analysers are separate scripts, fully compatible with Universum. It is possible to use them as independent Python modules. @@ -62,7 +63,6 @@ Pylint .. argparse:: :ref: universum.analyzers.pylint.pylint_argument_parser - :prog: {python} -m universum.analyzers.pylint Config example for ``universum.analyzers.pylint``: @@ -99,7 +99,6 @@ Mypy .. argparse:: :ref: universum.analyzers.mypy.mypy_argument_parser - :prog: {python} -m universum.analyzers.mypy Config example for ``universum.analyzers.mypy``: @@ -136,7 +135,6 @@ Uncrustify .. argparse:: :ref: universum.analyzers.uncrustify.uncrustify_argument_parser - :prog: {python} -m universum.analyzers.uncrustify :nodefault: Config example for ``universum.analyzers.uncrustify``: @@ -146,7 +144,7 @@ Config example for ``universum.analyzers.uncrustify``: from universum.configuration_support import Configuration, Step configs = Configuration([Step(name="uncrustify", code_report=True, command=[ - "{python}", "-m", "universum.analyzers.uncrustify", "--files", "/home/user/workspace/temp", + "{python}", "-m", "universum.analyzers.uncrustify", "--files", "/home/user/workspace/temp/*.c", "--cfg-file", "file_name.cfg", "--result-file", "${CODE_REPORT_FILE}", "--output-directory", "uncrustify" ])]) @@ -164,4 +162,38 @@ will produce this list of configurations: .. testoutput:: $ ./.universum.py - [{'name': 'uncrustify', 'code_report': True, 'command': '{python} -m universum.analyzers.uncrustify --files /home/user/workspace/temp --cfg-file file_name.cfg --result-file ${CODE_REPORT_FILE} --output-directory uncrustify'}] + [{'name': 'uncrustify', 'code_report': True, 'command': '{python} -m universum.analyzers.uncrustify --files /home/user/workspace/temp/*.c --cfg-file file_name.cfg --result-file ${CODE_REPORT_FILE} --output-directory uncrustify'}] + +Clang-format +------------ + +.. argparse:: + :ref: universum.analyzers.clang-format.clang_format_argument_parser + :nodefault: + +Config example for ``universum.analyzers.clang-format``: + +.. testcode:: + + from universum.configuration_support import Configuration, Step + + configs = Configuration([Step(name="clang-format", code_report=True, command=[ + "{python}", "-m", "universum.analyzers.clang-format", "--files", "/home/user/workspace/temp/*.c", + "--result-file", "${CODE_REPORT_FILE}", "--output-directory", "clang-format", "-e", "clang-format-15", + ])]) + + if __name__ == '__main__': + print(configs.dump()) + +will produce this list of configurations: + +.. testcode:: + :hide: + + print("$ ./.universum.py") + print(configs.dump()) + +.. testoutput:: + + $ ./.universum.py + [{'name': 'clang-format', 'code_report': True, 'command': '{python} -m universum.analyzers.clang-format --files /home/user/workspace/temp/*.c --result-file ${CODE_REPORT_FILE} --output-directory clang-format -e clang-format-15'}] diff --git a/pylintrc b/pylintrc index 580f2ef8..ee326fca 100644 --- a/pylintrc +++ b/pylintrc @@ -20,7 +20,8 @@ disable = missing-docstring, no-member, unsupported-assignment-operation, super-init-not-called, - wildcard-import + wildcard-import, + use-dict-literal [BASIC] no-docstring-rgx = .* @@ -36,4 +37,7 @@ max-parents = 10 max-line-length = 130 [SIMILARITIES] -ignore-imports=yes \ No newline at end of file +ignore-imports=yes + +[TYPECHECK] +signature-mutators=universum.analyzers.utils.analyzer diff --git a/setup.py b/setup.py index 642aa26c..da23d2c1 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,8 @@ def readme(): 'sh', 'lxml', 'typing-extensions', - 'ansi2html' + 'ansi2html', + 'pyyaml==6.0' ], extras_require={ 'p4': [p4], @@ -56,7 +57,8 @@ def readme(): 'coverage', 'mypy', 'types-requests', - 'selenium==3.141' + 'selenium==3.141', + 'types-PyYAML==6.0' ] }, package_data={'': ['*.css', '*.js']} diff --git a/tests/deployment_utils.py b/tests/deployment_utils.py index 4dee2a32..32a67194 100644 --- a/tests/deployment_utils.py +++ b/tests/deployment_utils.py @@ -2,10 +2,11 @@ import getpass import os +import shutil +import pathlib from pwd import getpwnam import docker -import py import pytest from requests.exceptions import ReadTimeout @@ -41,7 +42,8 @@ def add_bind_dirs(self, directories): if self._container: self.request.raiseerror("Container is already running, no dirs can be bound!") for directory in directories: - self._volumes[directory] = {'bind': directory, 'mode': 'rw'} + absolute_dir = str(pathlib.Path(directory).absolute()) + self._volumes[absolute_dir] = {'bind': absolute_dir, 'mode': 'rw'} def add_environment_variables(self, variables): if self._container: @@ -154,26 +156,27 @@ def clean_execution_environment(request): class LocalSources(utils.BaseVcsClient): - def __init__(self, root_directory: py.path.local, repo_file: py.path.local): + def __init__(self, root_directory: pathlib.Path, repo_file: pathlib.Path): super().__init__() self.root_directory = root_directory self.repo_file = repo_file @pytest.fixture() -def local_sources(tmpdir: py.path.local): +def local_sources(tmp_path: pathlib.Path): if utils.reuse_docker_containers(): - source_dir = py.path.local(".work") + source_dir = pathlib.Path(".work") try: - source_dir.remove(rec=1, ignore_errors=True) + shutil.rmtree(source_dir, ignore_errors=True) except OSError: pass - source_dir.ensure(dir=True) + source_dir.mkdir() else: - source_dir = tmpdir.mkdir("project_sources") - local_file = source_dir.join("readme.txt") - local_file.write("This is a an empty file") + source_dir = tmp_path / "project_sources" + source_dir.mkdir() + local_file = source_dir / "readme.txt" + local_file.write_text("This is a an empty file", encoding="utf-8") yield LocalSources(root_directory=source_dir, repo_file=local_file) diff --git a/tests/git_utils.py b/tests/git_utils.py index a4bb6cb5..2d02a2d5 100644 --- a/tests/git_utils.py +++ b/tests/git_utils.py @@ -1,11 +1,11 @@ # pylint: disable = redefined-outer-name import os +import pathlib from time import sleep import git from git.remote import RemoteProgress -import py import pytest import sh @@ -13,11 +13,11 @@ class GitServer: - def __init__(self, working_directory: py.path.local, branch_name: str): + def __init__(self, working_directory: pathlib.Path, branch_name: str): self.target_branch: str = branch_name self.target_file: str = "readme.txt" - self._working_directory: py.path.local = working_directory + self._working_directory: pathlib.Path = working_directory self._repo: git.Repo = git.Repo.init(working_directory) self._repo.daemon_export = True self._daemon_started: bool = False @@ -42,8 +42,8 @@ def std_redirect(line): with self._repo.config_writer() as configurator: configurator.set_value("user", "name", "Testing user") configurator.set_value("user", "email", "some@email.com") - self._file: py.path.local = self._working_directory.join(self.target_file) - self._file.write("") + self._file: pathlib.Path = self._working_directory / self.target_file + self._file.write_text("") self._repo.index.add([str(self._file)]) self._repo.index.commit("initial commit") @@ -57,7 +57,7 @@ def std_redirect(line): def make_a_change(self) -> str: self._branch.checkout() - self._file.write("One more line\n") + self._file.write_text("One more line\n") self._commit_count += 1 self._repo.index.add([str(self._file)]) @@ -68,8 +68,8 @@ def make_a_change(self) -> str: def commit_new_file(self) -> str: """ Make a mergeble commit """ self._commit_count += 1 - test_file = self._working_directory.join(f"test{self._commit_count}.txt") - test_file.write(f"Commit number #{self._commit_count}") + test_file = self._working_directory / f"test{self._commit_count}.txt" + test_file.write_text(f"Commit number #{self._commit_count}") self._repo.index.add([str(test_file)]) return str(self._repo.index.commit(f"Add file {self._commit_count}")) @@ -104,8 +104,9 @@ def exit(self) -> None: @pytest.fixture() -def git_server(tmpdir: py.path.local): - directory = tmpdir.mkdir("server") +def git_server(tmp_path: pathlib.Path): + directory = tmp_path / "server" + directory.mkdir() server = GitServer(directory, "testing") try: yield server @@ -114,7 +115,7 @@ def git_server(tmpdir: py.path.local): class GitClient(utils.BaseVcsClient): - def __init__(self, git_server: GitServer, directory: py.path.local): + def __init__(self, git_server: GitServer, directory: pathlib.Path): super().__init__() class Progress(RemoteProgress): @@ -123,9 +124,10 @@ def line_dropped(self, line): self.server: GitServer = git_server self.logger: Progress = Progress() - self.root_directory: py.path.local = directory.mkdir("client") + self.root_directory: pathlib.Path = directory / "client" + self.root_directory.mkdir() self.repo: git.Repo = git.Repo.clone_from(git_server.url, self.root_directory) - self.repo_file = self.root_directory.join(git_server.target_file) + self.repo_file = self.root_directory / git_server.target_file def get_last_change(self) -> str: changes = self.repo.git.log("origin/" + self.server.target_branch, pretty="oneline", max_count=1) @@ -144,13 +146,13 @@ def make_a_change(self) -> str: @pytest.fixture() -def git_client(git_server: GitServer, tmpdir: py.path.local): - yield GitClient(git_server, tmpdir) +def git_client(git_server: GitServer, tmp_path: pathlib.Path): + yield GitClient(git_server, tmp_path) class GitTestEnvironment(utils.BaseTestEnvironment): - def __init__(self, client: GitClient, directory: py.path.local, test_type: str): - db_file = directory.join("gitpoll.json") + def __init__(self, client: GitClient, directory: pathlib.Path, test_type: str): + db_file = directory / "gitpoll.json" super().__init__(client, directory, test_type, str(db_file)) self.vcs_client: GitClient diff --git a/tests/perforce_utils.py b/tests/perforce_utils.py index 53e628db..a570fc7b 100644 --- a/tests/perforce_utils.py +++ b/tests/perforce_utils.py @@ -1,9 +1,9 @@ # pylint: disable = redefined-outer-name, too-many-locals import time +import pathlib import docker -import py import pytest from P4 import P4, P4Exception from requests.exceptions import ReadTimeout @@ -177,12 +177,13 @@ def perforce_connection(request, docker_perforce: PerfoceDockerContainer): class PerforceWorkspace(utils.BaseVcsClient): - def __init__(self, connection: PerforceConnection, directory: py.path.local): + def __init__(self, connection: PerforceConnection, directory: pathlib.Path): super().__init__() - self.root_directory = directory.mkdir("workspace") - self.repo_file = self.root_directory.join("writeable_file.txt") + self.root_directory = directory / "workspace" + self.root_directory.mkdir() + self.repo_file = self.root_directory / "writeable_file.txt" - self.nonwritable_file: py.path.local = self.root_directory.join("usual_file.txt") + self.nonwritable_file: pathlib.Path = self.root_directory / "usual_file.txt" self.server: PerfoceDockerContainer = connection.server self.client_created: bool = False @@ -202,10 +203,10 @@ def setup(self) -> None: self.p4.run("add", str(self.nonwritable_file)) self.p4.run("edit", str(self.nonwritable_file)) - self.nonwritable_file.write("File " + str(self.nonwritable_file) + " has no special modifiers") + self.nonwritable_file.write_text("File " + str(self.nonwritable_file) + " has no special modifiers") self.p4.run("add", "-t", "+w", str(self.repo_file)) - self.repo_file.write("File " + str(self.repo_file) + " is always writable") + self.repo_file.write_text("File " + str(self.repo_file) + " is always writable") change = self.p4.run_change("-o")[0] change["Description"] = "Test submit" @@ -225,9 +226,9 @@ def setup(self) -> None: ]} self.p4.save_triggers(triggers) - def create_file(self, file_name: str) -> py.path.local: - p4_new_file = self.root_directory.join(file_name) - p4_new_file.write("This is unchanged line 1\nThis is unchanged line 2") + def create_file(self, file_name: str) -> pathlib.Path: + p4_new_file = self.root_directory / file_name + p4_new_file.write_text("This is unchanged line 1\nThis is unchanged line 2") self.p4.run("add", str(p4_new_file)) change = self.p4.run_change("-o")[0] @@ -241,14 +242,14 @@ def delete_file(self, file_name: str) -> None: change["Description"] = "Delete created file" self.p4.run_submit(change) - def shelve_file(self, file: py.path.local, content: str, shelve_cl=None) -> str: + def shelve_file(self, file: pathlib.Path, content: str, shelve_cl=None) -> str: if not shelve_cl: change = self.p4.fetch_change() change["Description"] = "This is a shelved CL" shelve_cl = self.p4.save_change(change)[0].split()[1] self.p4.run_edit("-c", shelve_cl, str(file)) - file.write(content) + file.write_text(content) self.p4.run_shelve("-fc", shelve_cl) self.p4.run_revert("-c", shelve_cl, str(file)) return shelve_cl @@ -274,7 +275,7 @@ def file_present(self, file_path: str) -> bool: def make_a_change(self) -> str: tmpfile = self.repo_file self.p4.run("edit", str(tmpfile)) - tmpfile.write("Change #1 " + str(tmpfile)) + tmpfile.write_text("Change #1 " + str(tmpfile)) change = self.p4.run_change("-o")[0] change["Description"] = "Test submit #1" @@ -293,8 +294,8 @@ def cleanup(self) -> None: @pytest.fixture() -def perforce_workspace(request, perforce_connection: PerforceConnection, tmpdir: py.path.local): - workspace = PerforceWorkspace(perforce_connection, tmpdir) +def perforce_workspace(request, perforce_connection: PerforceConnection, tmp_path: pathlib.Path): + workspace = PerforceWorkspace(perforce_connection, tmp_path) try: workspace.setup() yield workspace @@ -303,8 +304,8 @@ def perforce_workspace(request, perforce_connection: PerforceConnection, tmpdir: class P4TestEnvironment(utils.BaseTestEnvironment): - def __init__(self, perforce_workspace: PerforceWorkspace, directory: py.path.local, test_type: str): - db_file = directory.join("p4poll.json") + def __init__(self, perforce_workspace: PerforceWorkspace, directory: pathlib.Path, test_type: str): + db_file = directory / "p4poll.json" super().__init__(perforce_workspace, directory, test_type, str(db_file)) self.vcs_client: PerforceWorkspace @@ -332,4 +333,4 @@ def shelve_config(self, config: str) -> None: shelve_cl = self.vcs_client.shelve_file(self.vcs_client.repo_file, config) settings = self.settings settings.PerforceMainVcs.shelve_cls = [shelve_cl] - settings.Launcher.config_path = self.vcs_client.repo_file.basename + settings.Launcher.config_path = self.vcs_client.repo_file.name diff --git a/tests/test_api.py b/tests/test_api.py index 6b48624b..b41de1b2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -24,8 +24,8 @@ def test_p4_file_diff(docker_main_with_vcs: UniversumRunner): p4_file = docker_main_with_vcs.perforce.repo_file p4.run_edit(p4_file) - p4_file.write("This line is added to the file.\n") - p4.run_move(p4_file, p4_directory.join("some_new_file_name.txt")) + p4_file.write_text("This line is added to the file.\n") + p4.run_move(p4_file, p4_directory / "some_new_file_name.txt") change = p4.fetch_change() change["Description"] = "Rename basic config" shelve_cl = p4.save_change(change)[0].split()[1] @@ -49,8 +49,8 @@ def test_multiple_p4_file_diff(docker_main_with_vcs: UniversumRunner): p4_directory = docker_main_with_vcs.perforce.root_directory for index in range(0, 10000): - new_file = p4_directory.join(f"new_file_{index}.txt") - new_file.write(f"This is file #{index}\n") + new_file = p4_directory / f"new_file_{index}.txt" + new_file.write_text(f"This is file #{index}\n") p4.run_add(new_file) change = p4.fetch_change() change["Description"] = "Add 10000 files" @@ -77,7 +77,7 @@ def test_git_file_diff(docker_main_with_vcs: UniversumRunner): repo.git.checkout(server.target_branch) repo.git.checkout("new_testing_branch", b=True) - repo.git.mv(git_file, git_directory.join("some_new_file_name.txt")) + repo.git.mv(git_file, git_directory / "some_new_file_name.txt") change = repo.index.commit("Special commit for testing") repo.remotes.origin.push(progress=logger, all=True) @@ -104,8 +104,8 @@ def test_multiple_git_file_diff(docker_main_with_vcs: UniversumRunner): repo.git.checkout("new_testing_branch", b=True) files = [] for index in range(0, 10000): - new_file = git_directory.join(f"new_file_{index}.txt") - new_file.write(f"This is file #{index}\n") + new_file = git_directory / f"new_file_{index}.txt" + new_file.write_text(f"This is file #{index}\n") files.append(str(new_file)) repo.index.add(files) change = repo.index.commit("Special commit for testing") diff --git a/tests/test_argument_check.py b/tests/test_argument_check.py index 49344cf8..05974c80 100644 --- a/tests/test_argument_check.py +++ b/tests/test_argument_check.py @@ -1,4 +1,4 @@ -from typing import Union, List +from typing import Union, List, Optional import pytest from universum import __main__ @@ -101,7 +101,7 @@ def assert_incorrect_parameter(settings: ModuleNamespace, *args): def param(test_type: str, module: str, field: str, - vcs_type: Union[str, List[str]] = "*", error_match: str = None) -> None: + vcs_type: Union[str, List[str]] = "*", error_match: Optional[str] = None) -> None: if isinstance(vcs_type, list): for specific_vcs_type in vcs_type: diff --git a/tests/test_code_report.py b/tests/test_code_report.py index 9383afd9..f73cd2db 100644 --- a/tests/test_code_report.py +++ b/tests/test_code_report.py @@ -1,9 +1,9 @@ import inspect import os import re +import pathlib from typing import List -import py import pytest from . import utils @@ -16,7 +16,7 @@ def fixture_runner_with_analyzers(docker_main: UniversumRunner): docker_main.environment.install_python_module("pylint") docker_main.environment.install_python_module("mypy") - docker_main.environment.assert_successful_execution("apt install uncrustify") + docker_main.environment.assert_successful_execution("apt install -y uncrustify clang-format") yield docker_main @@ -128,6 +128,11 @@ def finalize(self) -> str: input_tab_size = 2 """ +config_clang_format = """ +--- +AllowShortFunctionsOnASingleLine: Empty +""" + log_fail = r'Found [0-9]+ issues' log_success = r'Issues not found' @@ -157,7 +162,7 @@ def test_code_report_direct_log(runner_with_analyzers: UniversumRunner, tested_c for idx, tested_content in enumerate(tested_contents): prelim_report = "report_file_" + str(idx) full_report = "${CODE_REPORT_FILE}" - runner_with_analyzers.local.root_directory.join(prelim_report).write(tested_content) + (runner_with_analyzers.local.root_directory / prelim_report).write_text(tested_content) config.add_cmd("Report " + str(idx), f"[\"bash\", \"-c\", \"cat './{prelim_report}' >> '{full_report}'\"]", step_config) log = runner_with_analyzers.run(config.finalize()) @@ -167,7 +172,10 @@ def test_code_report_direct_log(runner_with_analyzers: UniversumRunner, tested_c @pytest.mark.parametrize('analyzers, extra_args, tested_content, expected_success', [ [['uncrustify'], [], source_code_c, True], - [['uncrustify'], [], source_code_c.replace('\t', ' '), False], + [['uncrustify'], [], source_code_c.replace('\t', ' '), False], # by default uncrustify converts spaces to tabs + [['clang_format'], [], source_code_c.replace('\t', ' '), True], # by default clang-format expands tabs to 2 spaces + [['clang_format'], [], source_code_c.replace('\t', ' '), False], + [['clang_format', 'uncrustify'], [], source_code_c.replace('\t', ' '), False], [['pylint', 'mypy'], ["--python-version", python_version()], source_code_python, True], [['pylint'], ["--python-version", python_version()], source_code_python + '\n', False], [['mypy'], ["--python-version", python_version()], source_code_python.replace(': str', ': int'), False], @@ -177,6 +185,9 @@ def test_code_report_direct_log(runner_with_analyzers: UniversumRunner, tested_c ], ids=[ 'uncrustify_no_issues', 'uncrustify_found_issues', + 'clang_format_no_issues', + 'clang_format_found_issues', + 'clang_format_and_uncrustify_found_issues', 'pylint_and_mypy_both_no_issues', 'pylint_found_issues', 'mypy_found_issues', @@ -187,13 +198,16 @@ def test_code_report_log(runner_with_analyzers: UniversumRunner, analyzers, extr "--result-file", "${CODE_REPORT_FILE}", "--files", "source_file", ] - runner_with_analyzers.local.root_directory.join("source_file").write(tested_content) + (runner_with_analyzers.local.root_directory / "source_file").write_text(tested_content) config = ConfigData() for analyzer in analyzers: args = common_args + extra_args if analyzer == 'uncrustify': args += ["--cfg-file", "cfg"] - runner_with_analyzers.local.root_directory.join("cfg").write(config_uncrustify) + (runner_with_analyzers.local.root_directory / "cfg").write_text(config_uncrustify) + elif analyzer == 'clang_format': + (runner_with_analyzers.local.root_directory / ".clang-format").write_text(config_clang_format) + config.add_analyzer(analyzer, args) log = runner_with_analyzers.run(config.finalize()) @@ -210,7 +224,7 @@ def test_without_code_report_command(runner_with_analyzers: UniversumRunner): assert not pattern.findall(log) -@pytest.mark.parametrize('analyzer', ['pylint', 'mypy', 'uncrustify']) +@pytest.mark.parametrize('analyzer', ['pylint', 'mypy', 'uncrustify', 'clang_format']) @pytest.mark.parametrize('arg_set, expected_log', [ [["--files", "source_file.py"], "error: the following arguments are required: --result-file"], [["--files", "source_file.py", "--result-file"], "result-file: expected one argument"], @@ -235,59 +249,76 @@ def test_analyzer_python_version_params(runner_with_analyzers: UniversumRunner, "--result-file", "${CODE_REPORT_FILE}", '--rcfile'], "rcfile: expected one argument"], ['uncrustify', ["--files", "source_file", "--result-file", "${CODE_REPORT_FILE}"], - "Please specify the '--cfg_file' parameter or set an env. variable 'UNCRUSTIFY_CONFIG'"], + "Please specify the '--cfg-file' parameter or set 'UNCRUSTIFY_CONFIG' environment variable"], ['uncrustify', ["--files", "source_file", "--result-file", "${CODE_REPORT_FILE}", "--cfg-file", "cfg", "--output-directory", "."], - "Target and source folders for uncrustify are not allowed to match"], + "Target folder must not be identical to source folder"], + ['clang_format', ["--files", "source_file", "--result-file", "${CODE_REPORT_FILE}", "--output-directory", "."], + "Target folder must not be identical to source folder"], ]) def test_analyzer_specific_params(runner_with_analyzers: UniversumRunner, analyzer, arg_set, expected_log): - source_file = runner_with_analyzers.local.root_directory.join("source_file") - source_file.write(source_code_python) + source_file = runner_with_analyzers.local.root_directory / "source_file" + source_file.write_text(source_code_python) log = runner_with_analyzers.run(ConfigData().add_analyzer(analyzer, arg_set).finalize()) assert re.findall(fr'Run {analyzer} - [^\n]*Failed', log), f"'{analyzer}' info is not found in '{log}'" assert expected_log in log, f"'{expected_log}' is not found in '{log}'" -@pytest.mark.parametrize('extra_args, tested_content, expected_success, expected_artifact', [ - [[], source_code_c, True, False], - [["--report-html"], source_code_c.replace('\t', ' '), False, True], - [[], source_code_c.replace('\t', ' '), False, False], +@pytest.mark.parametrize('analyzer, extra_args, tested_content, expected_success, expected_artifact', [ + ['uncrustify', ["--report-html"], source_code_c, True, False], + ['uncrustify', ["--report-html"], source_code_c.replace('\t', ' '), False, True], + ['uncrustify', [], source_code_c.replace('\t', ' '), False, False], + ['clang_format', ["--report-html"], source_code_c.replace('\t', ' '), True, False], + ['clang_format', ["--report-html"], source_code_c, False, True], + ['clang_format', [], source_code_c, False, False], ], ids=[ "uncrustify_html_file_not_needed", "uncrustify_html_file_saved", "uncrustify_html_file_disabled", + "clang_format_html_file_not_needed", + "clang_format_html_file_saved", + "clang_format_html_file_disabled", ]) -def test_uncrustify_file_diff(runner_with_analyzers: UniversumRunner, - extra_args, tested_content, expected_success, expected_artifact): +def test_diff_html_file(runner_with_analyzers: UniversumRunner, analyzer, + extra_args, tested_content, expected_success, expected_artifact): + root = runner_with_analyzers.local.root_directory - source_file = root.join("source_file") - source_file.write(tested_content) - root.join("cfg").write(config_uncrustify) + source_file = root / "source_file" + source_file.write_text(tested_content) common_args = [ "--result-file", "${CODE_REPORT_FILE}", "--files", "source_file", - "--cfg-file", "cfg", + "--output-directory", "diff_temp" ] + if analyzer == 'uncrustify': + (root / "cfg").write_text(config_uncrustify) + common_args.extend(["--cfg-file", "cfg"]) + elif analyzer == 'clang_format': + (root / ".clang-format").write_text(config_clang_format) + args = common_args + extra_args - extra_config = "artifacts='./uncrustify/source_file.html'" - log = runner_with_analyzers.run(ConfigData().add_analyzer('uncrustify', args, extra_config).finalize()) + extra_config = "artifacts='./diff_temp/source_file.html'" + log = runner_with_analyzers.run(ConfigData().add_analyzer(analyzer, args, extra_config).finalize()) expected_log = log_success if expected_success else log_fail assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" expected_artifacts_state = "Success" if expected_artifact else "Failed" - expected_log = f"Collecting artifacts for the 'Run uncrustify' step - [^\n]*{expected_artifacts_state}" + expected_log = f"Collecting artifacts for the 'Run {analyzer}' step - [^\n]*{expected_artifacts_state}" assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" -def test_code_report_extended_arg_search(tmpdir: py.path.local, stdout_checker: FuzzyCallChecker): - env = utils.LocalTestEnvironment(tmpdir, "main") +def test_code_report_extended_arg_search(tmp_path: pathlib.Path, stdout_checker: FuzzyCallChecker): + """ + Test if ${CODE_REPORT_FILE} is replaced not only in --result-file argument of the Step + """ + env = utils.LocalTestEnvironment(tmp_path, "main") env.settings.Vcs.type = "none" - env.settings.LocalMainVcs.source_dir = str(tmpdir) + env.settings.LocalMainVcs.source_dir = str(tmp_path) - source_file = tmpdir.join("source_file.py") - source_file.write(source_code_python + '\n') + source_file = tmp_path / "source_file.py" + source_file.write_text(source_code_python + '\n') config = f""" from universum.configuration_support import Configuration @@ -297,20 +328,20 @@ def test_code_report_extended_arg_search(tmpdir: py.path.local, stdout_checker: --python-version {python_version()} --files {str(source_file)}'])]) """ - env.configs_file.write(config) + env.configs_file.write_text(config) env.run() stdout_checker.assert_has_calls_with_param(log_fail, is_regexp=True) assert os.path.exists(os.path.join(env.settings.ArtifactCollector.artifact_dir, "Run_static_pylint.json")) -def test_code_report_extended_arg_search_embedded(tmpdir: py.path.local, stdout_checker: FuzzyCallChecker): - env = utils.LocalTestEnvironment(tmpdir, "main") +def test_code_report_extended_arg_search_embedded(tmp_path: pathlib.Path, stdout_checker: FuzzyCallChecker): + env = utils.LocalTestEnvironment(tmp_path, "main") env.settings.Vcs.type = "none" - env.settings.LocalMainVcs.source_dir = str(tmpdir) + env.settings.LocalMainVcs.source_dir = str(tmp_path) - source_file = tmpdir.join("source_file.py") - source_file.write(source_code_python + '\n') + source_file = tmp_path / "source_file.py" + source_file.write_text(source_code_python + '\n') config = """ from universum.configuration_support import Configuration, Step @@ -322,7 +353,7 @@ def test_code_report_extended_arg_search_embedded(tmpdir: py.path.local, stdout_ ]) """ - env.configs_file.write(config) + env.configs_file.write_text(config) env.run() stdout_checker.assert_absent_calls_with_param("${CODE_REPORT_FILE}") diff --git a/tests/test_create_config.py b/tests/test_create_config.py index 618b1d57..bdd697bf 100644 --- a/tests/test_create_config.py +++ b/tests/test_create_config.py @@ -1,12 +1,12 @@ import os import subprocess -import py +import pathlib from .utils import python -def test_create_config(tmpdir: py.path.local): - launch_parameters = dict(capture_output=True, cwd=tmpdir, env=dict(os.environ, PYTHONPATH=os.getcwd())) +def test_create_config(tmp_path: pathlib.Path): + launch_parameters = dict(capture_output=True, cwd=tmp_path, env=dict(os.environ, PYTHONPATH=os.getcwd())) result = subprocess.run([python(), "-m", "universum", "init"], check=True, **launch_parameters) # type: ignore new_command = '' for line in result.stdout.splitlines(): diff --git a/tests/test_deployment.py b/tests/test_deployment.py index 86f652a2..2b4c07a0 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -9,17 +9,17 @@ def test_minimal_install(clean_docker_main: UniversumRunner): # Run locally log = clean_docker_main.run(simple_test_config, force_installed=True) - assert clean_docker_main.local.repo_file.basename in log + assert clean_docker_main.local.repo_file.name in log # Run from Git clean_docker_main.clean_artifacts() log = clean_docker_main.run(simple_test_config, vcs_type="git", force_installed=True) - assert clean_docker_main.git.repo_file.basename in log + assert clean_docker_main.git.repo_file.name in log # Run from P4 clean_docker_main.clean_artifacts() log = clean_docker_main.run(simple_test_config, vcs_type="p4", force_installed=True) - assert clean_docker_main.perforce.repo_file.basename in log + assert clean_docker_main.perforce.repo_file.name in log def test_minimal_install_with_git_only(clean_docker_main_no_p4: UniversumRunner, capsys): @@ -30,7 +30,7 @@ def test_minimal_install_with_git_only(clean_docker_main_no_p4: UniversumRunner, # Run from git clean_docker_main_no_p4.clean_artifacts() log = clean_docker_main_no_p4.run(simple_test_config, vcs_type="git", force_installed=True) - assert clean_docker_main_no_p4.git.repo_file.basename in log + assert clean_docker_main_no_p4.git.repo_file.name in log def test_minimal_install_plain_ubuntu(clean_docker_main_no_vcs: UniversumRunner, capsys): @@ -44,4 +44,4 @@ def test_minimal_install_plain_ubuntu(clean_docker_main_no_vcs: UniversumRunner, # Run locally log = clean_docker_main_no_vcs.run(simple_test_config, force_installed=True) - assert clean_docker_main_no_vcs.local.repo_file.basename in log + assert clean_docker_main_no_vcs.local.repo_file.name in log diff --git a/tests/test_git_poll.py b/tests/test_git_poll.py index 437014fe..df6f96df 100644 --- a/tests/test_git_poll.py +++ b/tests/test_git_poll.py @@ -1,6 +1,7 @@ # pylint: disable = redefined-outer-name -import py +from typing import Optional +import pathlib import pytest from .conftest import FuzzyCallChecker @@ -9,11 +10,12 @@ @pytest.fixture() -def git_poll_environment(git_client: GitClient, tmpdir: py.path.local): - yield GitTestEnvironment(git_client, tmpdir, test_type="poll") +def git_poll_environment(git_client: GitClient, tmp_path: pathlib.Path): + yield GitTestEnvironment(git_client, tmp_path, test_type="poll") -def make_branch_with_changes(git_server: GitServer, branch_name: str, commits_number: int, branch_from: str = None): +def make_branch_with_changes(git_server: GitServer, branch_name: str, commits_number: int, + branch_from: Optional[str] = None): """ Creates a branch from the current or specified (by name) and adds passed commits number. diff --git a/tests/test_github_handler.py b/tests/test_github_handler.py index f1252c32..39e6b842 100644 --- a/tests/test_github_handler.py +++ b/tests/test_github_handler.py @@ -1,7 +1,7 @@ # pylint: disable = redefined-outer-name, abstract-method +import pathlib import pytest -import py from universum.modules.vcs.github_vcs import GithubToken from .conftest import FuzzyCallChecker @@ -45,7 +45,7 @@ class GithubHandlerEnvironment(BaseTestEnvironment): }""" check_run_url: str = "http://localhost/" - def __init__(self, directory: py.path.local): + def __init__(self, directory: pathlib.Path): super().__init__(BaseVcsClient(), directory, "github-handler", "") self.settings.GithubHandler.payload = "{}" @@ -57,8 +57,8 @@ def __init__(self, directory: py.path.local): @pytest.fixture() -def github_handler_environment(tmpdir: py.path.local): - yield GithubHandlerEnvironment(tmpdir) +def github_handler_environment(tmp_path: pathlib.Path): + yield GithubHandlerEnvironment(tmp_path) @pytest.fixture(autouse=True) diff --git a/tests/test_html_output.py b/tests/test_html_output.py index d2ca0825..26031463 100644 --- a/tests/test_html_output.py +++ b/tests/test_html_output.py @@ -45,16 +45,16 @@ log_name = "universum_log" -def create_environment(test_type, tmpdir): - env = utils.LocalTestEnvironment(tmpdir, test_type) - env.configs_file.write(config) +def create_environment(test_type, tmp_path): + env = utils.LocalTestEnvironment(tmp_path, test_type) + env.configs_file.write_text(config) env.settings.Output.html_log = log_name return env @pytest.fixture(params=["main", "nonci"]) -def environment_main_and_nonci(request, tmpdir): - yield create_environment(request.param, tmpdir) +def environment_main_and_nonci(request, tmp_path): + yield create_environment(request.param, tmp_path) @pytest.fixture() @@ -66,19 +66,19 @@ def browser(): firefox.close() -def test_cli_log_custom_name(tmpdir): +def test_cli_log_custom_name(tmp_path): custom_log_name = "custom_name.html" - artifact_dir = check_cli(tmpdir, ["-hl", custom_log_name]) + artifact_dir = check_cli(tmp_path, ["-hl", custom_log_name]) assert os.path.exists(os.path.join(artifact_dir, custom_log_name)) -def test_cli_log_default_name(tmpdir): - artifact_dir = check_cli(tmpdir, ["-hl"]) +def test_cli_log_default_name(tmp_path): + artifact_dir = check_cli(tmp_path, ["-hl"]) assert os.path.exists(os.path.join(artifact_dir, "universum_log.html")) -def test_cli_no_log_requested(tmpdir): - artifact_dir = check_cli(tmpdir, []) +def test_cli_no_log_requested(tmp_path): + artifact_dir = check_cli(tmp_path, []) for file_name in os.listdir(artifact_dir): assert not file_name.endswith(".html") @@ -88,8 +88,8 @@ def test_success(environment_main_and_nonci, browser): check_html_log(environment_main_and_nonci.artifact_dir, browser) -def test_success_clean_build(tmpdir, browser): - env = create_environment("main", tmpdir) +def test_success_clean_build(tmp_path, browser): + env = create_environment("main", tmp_path) env.settings.Main.clean_build = True env.run() check_html_log(env.artifact_dir, browser) @@ -298,13 +298,13 @@ def check_timestamps(body_element, universum_log_element): assert delta.seconds <= 60 -def check_cli(tmpdir, html_log_params): - artifact_dir = tmpdir.join("artifacts") - config_file = tmpdir.join("configs.py") +def check_cli(tmp_path, html_log_params): + artifact_dir = tmp_path / "artifacts" + config_file = tmp_path / "configs.py" config_file.write_text(config, "utf-8") cli_params = ["-vt", "none", - "-fsd", str(tmpdir), + "-fsd", str(tmp_path), "-cfg", str(config_file), "-ad", str(artifact_dir)] html_log_params.extend(cli_params) diff --git a/tests/test_integration.py b/tests/test_integration.py index 6b93202a..11c567cc 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -2,9 +2,10 @@ import signal import subprocess import time +import shutil +import pathlib from typing import Any -import py import pytest from .deployment_utils import UniversumRunner, LocalSources @@ -20,7 +21,7 @@ def get_line_with_text(text: str, log: str) -> str: def test_minimal_execution(docker_main_and_nonci: UniversumRunner): log = docker_main_and_nonci.run(simple_test_config) - assert docker_main_and_nonci.local.repo_file.basename in log + assert docker_main_and_nonci.local.repo_file.name in log def test_artifacts(docker_main: UniversumRunner): @@ -223,12 +224,12 @@ def test_empty_steps(docker_main_and_nonci: UniversumRunner): def test_minimal_git(docker_main_with_vcs: UniversumRunner): log = docker_main_with_vcs.run(simple_test_config, vcs_type="git") - assert docker_main_with_vcs.git.repo_file.basename in log + assert docker_main_with_vcs.git.repo_file.name in log def test_minimal_p4(docker_main_with_vcs: UniversumRunner): log = docker_main_with_vcs.run(simple_test_config, vcs_type="p4") - assert docker_main_with_vcs.perforce.repo_file.basename in log + assert docker_main_with_vcs.perforce.repo_file.name in log def test_p4_params(docker_main_with_vcs: UniversumRunner): @@ -237,20 +238,20 @@ def test_p4_params(docker_main_with_vcs: UniversumRunner): config = f""" from universum.configuration_support import Configuration, Step -configs = Configuration([Step(name="Test step", command=["cat", "{p4_file.basename}"])]) +configs = Configuration([Step(name="Test step", command=["cat", "{p4_file.name}"])]) """ # Prepare SYNC_CHANGELIST sync_cl = p4.run_changes("-s", "submitted", "-m1", docker_main_with_vcs.perforce.depot)[0]["change"] p4.run_edit(docker_main_with_vcs.perforce.depot) - p4_file.write("This line shouldn't be in file.\n") + p4_file.write_text("This line shouldn't be in file.\n") change = p4.fetch_change() change["Description"] = "Rename basic config" p4.run_submit(change) # Prepare SHELVE_CHANGELIST p4.run_edit(docker_main_with_vcs.perforce.depot) - p4_file.write("This line should be in file.\n") + p4_file.write_text("This line should be in file.\n") change = p4.fetch_change() change["Description"] = "CL for shelving" shelve_cl = p4.save_change(change)[0].split()[1] @@ -305,8 +306,8 @@ def test_empty_required_params(docker_main_with_vcs: UniversumRunner, url_error_ def test_environment(docker_main_and_nonci: UniversumRunner): - script = docker_main_and_nonci.local.root_directory.join("script.sh") - script.write("""#!/bin/bash + script = docker_main_and_nonci.local.root_directory / "script.sh" + script.write_text("""#!/bin/bash echo ${SPECIAL_TESTING_VARIABLE} """) script.chmod(0o777) @@ -333,19 +334,19 @@ def test_environment(docker_main_and_nonci: UniversumRunner): @pytest.mark.parametrize("terminate_type", [signal.SIGINT, signal.SIGTERM], ids=["interrupt", "terminate"]) -def test_abort(local_sources: LocalSources, tmpdir: py.path.local, terminate_type): +def test_abort(local_sources: LocalSources, tmp_path: pathlib.Path, terminate_type): config = """ from universum.configuration_support import Configuration configs = Configuration([dict(name="Long step", command=["sleep", "10"])]) * 5 """ - config_file = tmpdir.join("configs.py") - config_file.write(config) + config_file = tmp_path / "configs.py" + config_file.write_text(config) with subprocess.Popen([python(), "-m", "universum", "-o", "console", "-st", "local", "-vt", "none", - "-pr", str(tmpdir.join("project_root")), - "-ad", str(tmpdir.join("artifacts")), + "-pr", str(tmp_path / "project_root"), + "-ad", str(tmp_path / "artifacts"), "-fsd", str(local_sources.root_directory), "-cfg", str(config_file)]) as process: time.sleep(5) @@ -353,30 +354,30 @@ def test_abort(local_sources: LocalSources, tmpdir: py.path.local, terminate_typ assert process.wait(5) == 3 -def test_exit_code(local_sources: LocalSources, tmpdir: py.path.local): +def test_exit_code(local_sources: LocalSources, tmp_path: pathlib.Path): config = """ from universum.configuration_support import Configuration configs = Configuration([dict(name="Unsuccessful step", command=["exit", "1"])]) """ - config_file = tmpdir.join("configs.py") - config_file.write(config) + config_file = tmp_path / "configs.py" + config_file.write_text(config) with subprocess.Popen([python(), "-m", "universum", "-o", "console", "-st", "local", "-vt", "none", - "-pr", str(tmpdir.join("project_root")), - "-ad", str(tmpdir.join("artifacts")), + "-pr", str(tmp_path / "project_root"), + "-ad", str(tmp_path / "artifacts"), "-fsd", str(local_sources.root_directory), "-cfg", str(config_file)]) as process: assert process.wait() == 0 - tmpdir.join("artifacts").remove(rec=True) + artifacts_dir = tmp_path / "artifacts" + shutil.rmtree(str(artifacts_dir)) with subprocess.Popen([python(), "-m", "universum", "--fail-unsuccessful", "-o", "console", "-st", "local", "-vt", "none", - "-pr", str(tmpdir.join("project_root")), - "-ad", str(tmpdir.join("artifacts")), + "-pr", str(tmp_path / "project_root"), + "-ad", str(tmp_path / "artifacts"), "-fsd", str(local_sources.root_directory), "-cfg", str(config_file)]) as process: assert process.wait() == 1 - diff --git a/tests/test_nonci.py b/tests/test_nonci.py index 7d33eb61..d03bb4f0 100644 --- a/tests/test_nonci.py +++ b/tests/test_nonci.py @@ -23,7 +23,7 @@ def test_launcher_output(docker_nonci: UniversumRunner): - version control and review system are not used - project root is set to current directory """ - cwd = docker_nonci.local.root_directory.strpath + cwd = str(docker_nonci.local.root_directory.absolute()) artifacts = docker_nonci.artifact_dir file_output_expected = f"Adding file {artifacts}/test_step_log.txt to artifacts" pwd_string_in_logs = f"pwd:[{cwd}]" diff --git a/tests/test_p4_exception_handling.py b/tests/test_p4_exception_handling.py index 5b510fa7..d88bd2ff 100644 --- a/tests/test_p4_exception_handling.py +++ b/tests/test_p4_exception_handling.py @@ -1,6 +1,8 @@ # pylint: disable = redefined-outer-name -import py +import os +import shutil +import pathlib import pytest from universum import __main__ @@ -9,8 +11,8 @@ @pytest.fixture() -def perforce_environment(perforce_workspace: PerforceWorkspace, tmpdir: py.path.local): - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="main") +def perforce_environment(perforce_workspace: PerforceWorkspace, tmp_path: pathlib.Path): + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="main") def test_p4_forbidden_local_revert(perforce_environment: P4TestEnvironment, stdout_checker: FuzzyCallChecker): @@ -26,8 +28,8 @@ def test_p4_forbidden_local_revert(perforce_environment: P4TestEnvironment, stdo perforce_environment.shelve_config(config) result = __main__.run(perforce_environment.settings) # Clean up the directory at once to make sure it doesn't remain non-writable even if some assert fails - perforce_environment.temp_dir.chmod(0o0777, rec=1) - perforce_environment.temp_dir.remove(rec=1) + os.system(f"chmod -R 777 {perforce_environment.temp_dir}") + shutil.rmtree(str(perforce_environment.temp_dir)) assert result == 0 diff --git a/tests/test_p4_revert_unshelved.py b/tests/test_p4_revert_unshelved.py index a2253932..997876a9 100644 --- a/tests/test_p4_revert_unshelved.py +++ b/tests/test_p4_revert_unshelved.py @@ -25,7 +25,7 @@ def __init__(self, perforce_workspace: PerforceWorkspace): settings.PerforceVcs.user = perforce_workspace.server.user settings.PerforceVcs.password = perforce_workspace.server.password settings.Output.type = "term" - settings.ProjectDirectory.project_root = str(self.perforce_workspace.root_directory + "/../new_workspace") + settings.ProjectDirectory.project_root = str(self.perforce_workspace.root_directory) + "/../new_workspace" settings.PerforceWithMappings.mappings = [self.perforce_workspace.depot + " /..."] settings.PerforceMainVcs.force_clean = True settings.PerforceMainVcs.client = "new_client" @@ -61,13 +61,14 @@ def diff_parameters(perforce_workspace: PerforceWorkspace): def test_p4_c_and_revert(diff_parameters): # pylint: disable = too-many-locals p4 = diff_parameters.perforce_workspace.p4 - test_dir = diff_parameters.perforce_workspace.root_directory.ensure("test_files", dir=True) + test_dir = diff_parameters.perforce_workspace.root_directory / "test_files" + test_dir.mkdir() def create_file(filename): - cur_file = test_dir.join(filename) + cur_file = test_dir / filename p4.run("add", str(cur_file)) p4.run("edit", str(cur_file)) - cur_file.write(f"import os\n\nprint('File {0} has no special modifiers.')\n" + cur_file.write_text(f"import os\n\nprint('File {0} has no special modifiers.')\n" f"print(os.name)\nprint(os.getcwd())\nprint(os.strerror(3))\n" f"print('File change type: {filename}')\n") @@ -84,27 +85,27 @@ def create_file(filename): p4.run_submit(change) # open for edit for edit, move/add and move/rename files - for cur_file in [test_dir.join("edit"), test_dir.join("move"), test_dir.join("rename")]: + for cur_file in [test_dir / "edit", test_dir / "move", test_dir / "rename"]: p4.run("edit", str(cur_file)) # edit - test_dir.join("edit").write(f"import os\n\nprint('File {test_dir.join('edit')} has no special modifiers.')\n" + (test_dir / "edit").write_text(f"import os\n\nprint('File {test_dir / 'edit'} has no special modifiers.')\n" f"print(os.name)\nprint(os.getegid())\nprint(os.ctermid())\n" f"print('File change type: \"edit\"')\n") # move, rename - p4.run("move", test_dir + "/move", test_dir + "/moved/move") - p4.run("rename", test_dir + "/rename", test_dir + "/renamed_rename") + p4.run("move", test_dir / "move", test_dir / "moved/move") + p4.run("rename", test_dir / "rename", test_dir / "renamed_rename") # integrate - p4.run("integrate", test_dir + "/integrate", test_dir + "/integrated") + p4.run("integrate", test_dir / "integrate", test_dir / "integrated") p4.run("resolve", "-at") # branch - p4.run("integrate", test_dir + "/branch", test_dir + "/moved/branch_to") + p4.run("integrate", test_dir / "branch", test_dir / "moved/branch_to") # delete - p4.run("delete", test_dir + "/delete") + p4.run("delete", test_dir / "delete") # add - add = test_dir.join("add") - add.write("\nprint('new file')\nprint('not in repo, only for shelve.')") + add = test_dir / "add" + add.write_text("\nprint('new file')\nprint('not in repo, only for shelve.')") p4.run("add", add) # make change diff --git a/tests/test_p4_submit.py b/tests/test_p4_submit.py index 83675ac2..dee0bba6 100644 --- a/tests/test_p4_submit.py +++ b/tests/test_p4_submit.py @@ -1,6 +1,6 @@ # pylint: disable = redefined-outer-name -import py +import pathlib import pytest from . import utils @@ -8,8 +8,8 @@ @pytest.fixture() -def p4_submit_environment(perforce_workspace: PerforceWorkspace, tmpdir: py.path.local): - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="submit") +def p4_submit_environment(perforce_workspace: PerforceWorkspace, tmp_path: pathlib.Path): + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="submit") def test_fail_changing_non_checked_out_file(p4_submit_environment: P4TestEnvironment): @@ -28,7 +28,7 @@ def test_success_changing_checked_out_file(p4_submit_environment: P4TestEnvironm p4_submit_environment.vcs_client.p4.run("edit", str(target_file)) text = utils.randomize_name("This is change ") - target_file.write(text + "\n") + target_file.write_text(text + "\n") change = p4_submit_environment.vcs_client.p4.run_change("-o")[0] change["Description"] = "Test submit" diff --git a/tests/test_poll.py b/tests/test_poll.py index 7f1a53c7..1b6d4d44 100644 --- a/tests/test_poll.py +++ b/tests/test_poll.py @@ -1,7 +1,7 @@ # pylint: disable = redefined-outer-name from typing import Union -import py +import pathlib import pytest from universum import __main__ @@ -11,15 +11,15 @@ from .utils import LocalTestEnvironment -def test_poll_local_vcs(tmpdir: py.path.local): - env = LocalTestEnvironment(tmpdir, "poll") +def test_poll_local_vcs(tmp_path: pathlib.Path): + env = LocalTestEnvironment(tmp_path, "poll") env.run() def test_p4_success_command_line_no_changes(stdout_checker: FuzzyCallChecker, perforce_workspace: PerforceWorkspace, - tmpdir: py.path.local): - db_file = tmpdir.join("p4poll.json") + tmp_path: pathlib.Path): + db_file = tmp_path / "p4poll.json" result = __main__.main(["poll", "-ot", "term", "-vt", "p4", "-f", str(db_file), @@ -34,8 +34,8 @@ def test_p4_success_command_line_no_changes(stdout_checker: FuzzyCallChecker, def test_git_success_command_line_no_changes(stdout_checker: FuzzyCallChecker, git_server: GitServer, - tmpdir: py.path.local): - db_file = tmpdir.join("gitpoll.json") + tmp_path: pathlib.Path): + db_file = tmp_path / "gitpoll.json" result = __main__.main(["poll", "-ot", "term", "-vt", "git", "-f", str(db_file), @@ -48,8 +48,8 @@ def test_git_success_command_line_no_changes(stdout_checker: FuzzyCallChecker, def test_p4_error_command_line_wrong_port(stdout_checker: FuzzyCallChecker, perforce_workspace: PerforceWorkspace, - tmpdir: py.path.local): - db_file = tmpdir.join("p4poll.json") + tmp_path: pathlib.Path): + db_file = tmp_path / "p4poll.json" result = __main__.main(["poll", "-ot", "term", "-vt", "p4", "-f", str(db_file), @@ -64,8 +64,8 @@ def test_p4_error_command_line_wrong_port(stdout_checker: FuzzyCallChecker, def test_git_error_command_line_wrong_port(stdout_checker: FuzzyCallChecker, git_server: GitServer, - tmpdir: py.path.local): - db_file = tmpdir.join("gitpoll.json") + tmp_path: pathlib.Path): + db_file = tmp_path / "gitpoll.json" result = __main__.main(["poll", "-ot", "term", "-vt", "git", "-f", str(db_file), @@ -77,11 +77,11 @@ def test_git_error_command_line_wrong_port(stdout_checker: FuzzyCallChecker, @pytest.fixture(params=["git", "p4"]) -def poll_environment(request, perforce_workspace: PerforceWorkspace, git_client: GitClient, tmpdir: py.path.local): +def poll_environment(request, perforce_workspace: PerforceWorkspace, git_client: GitClient, tmp_path: pathlib.Path): if request.param == "git": - yield GitTestEnvironment(git_client, tmpdir, test_type="poll") + yield GitTestEnvironment(git_client, tmp_path, test_type="poll") else: - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="poll") + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="poll") def test_error_one_change(stdout_checker: FuzzyCallChecker, log_exception_checker: FuzzyCallChecker, diff --git a/tests/test_preprocess_artifacts.py b/tests/test_preprocess_artifacts.py new file mode 100644 index 00000000..bca3fa40 --- /dev/null +++ b/tests/test_preprocess_artifacts.py @@ -0,0 +1,191 @@ +# pylint: disable = redefined-outer-name + +import inspect +import pathlib +import zipfile +from typing import Generator + +import pytest + +from .utils import LocalTestEnvironment +from .conftest import FuzzyCallChecker + + +class ArtifactsTestEnvironment(LocalTestEnvironment): + + def __init__(self, tmp_path: pathlib.Path, test_type: str) -> None: + super().__init__(tmp_path, test_type) + self.artifact_name: str = "artifact" + self.artifact_path: pathlib.Path = self.artifact_dir / self.artifact_name + self.artifact_name_with_suffix = f"{self.artifact_name}_suffix" + self.artifact_path_with_suffix: pathlib.Path = self.artifact_dir / self.artifact_name_with_suffix + self.artifact_content: str = "artifact content" + self.dir_name: str = "artifacts_test_dir" + self.dir_archive: pathlib.Path = self.artifact_dir / f"{self.dir_name}.zip" + self.artifact_in_dir: pathlib.Path = self.artifact_dir / self.dir_name / self.artifact_name + + def write_config_file(self, artifact_prebuild_clean: bool, is_report_artifact: bool = False) -> None: + artifact_in_dir: str = f"{self.dir_name}/{self.artifact_name}" + artifacts_key: str = "report_artifacts" if is_report_artifact else "artifacts" + config: str = inspect.cleandoc(f""" + from universum.configuration_support import Configuration, Step + step_with_file = Step( + name='Step with file', + command=['bash', '-c', 'echo "{self.artifact_content}" > {self.artifact_name}'], + {artifacts_key}='{self.artifact_name}', + artifact_prebuild_clean={artifact_prebuild_clean}) + step_with_dir = Step( + name='Step with directory', + command=['bash', '-c', 'mkdir {self.dir_name}; echo "{self.artifact_content}" > {artifact_in_dir}'], + {artifacts_key}='{self.dir_name}', + artifact_prebuild_clean={artifact_prebuild_clean}) + configs = Configuration([step_with_file, step_with_dir]) + """) + self.store_config_to_file(config) + + def write_config_file_wildcard(self, artifact_prebuild_clean: bool) -> None: + config: str = inspect.cleandoc(f""" + from universum.configuration_support import Configuration, Step + step = Step( + name='Step', + command=['bash', '-c', + 'echo "{self.artifact_content}" > {self.artifact_name};' + 'echo "{self.artifact_content}" > {self.artifact_name_with_suffix}'], + artifacts='{self.artifact_name}*', + artifact_prebuild_clean={artifact_prebuild_clean}) + configs = Configuration([step]) + """) + self.store_config_to_file(config) + + def store_config_to_file(self, config: str): + self.configs_file.write_text(config, "utf-8") + + def create_artifact_file(self, directory: pathlib.Path, file_name: str, is_zip: bool = False) -> None: + artifact_name: str = f"{file_name}.zip" if is_zip else file_name + precreated_artifact: pathlib.Path = directory / artifact_name + with open(precreated_artifact, "w", encoding="utf-8") as f: + f.write("pre-created artifact content") + + def create_artifacts_dir(self, directory: pathlib.Path) -> None: + precreated_artifacts_dir: pathlib.Path = directory / self.dir_name + precreated_artifacts_dir.mkdir() + self.create_artifact_file(precreated_artifacts_dir, self.artifact_name) + + def check_artifact_present(self, path: pathlib.Path) -> None: + assert path.exists() + with open(path, encoding="utf-8") as f: + content: str = f.read().replace("\n", "") + assert content == self.artifact_content + + def check_artifact_absent(self) -> None: + assert not self.artifact_path.exists() + + def check_dir_zip_artifact_present(self) -> None: + assert self.dir_archive.exists() + with zipfile.ZipFile(self.dir_archive) as dir_zip: + assert self.artifact_name in dir_zip.namelist() + with dir_zip.open(self.artifact_name) as f: + content: str = f.read().decode(encoding="utf-8").replace("\n", "") + assert content == self.artifact_content + + def check_dir_zip_artifact_absent(self) -> None: + assert not self.dir_archive.exists() + + +@pytest.fixture() +def test_env(tmp_path: pathlib.Path) -> Generator[ArtifactsTestEnvironment, None, None]: + yield ArtifactsTestEnvironment(tmp_path, "main") + + +@pytest.mark.parametrize("prebuild_clean", [True, False]) +def test_no_artifact(test_env: ArtifactsTestEnvironment, + prebuild_clean: bool) -> None: + test_env.write_config_file(artifact_prebuild_clean=prebuild_clean) + test_env.run() + test_env.check_artifact_present(test_env.artifact_path) + + +@pytest.mark.parametrize("test_type", ["main", "nonci"]) +@pytest.mark.parametrize("is_report_artifact", [True, False]) +def test_artifact_in_sources_prebuild_clean(tmp_path: pathlib.Path, + is_report_artifact: bool, + test_type: str) -> None: + test_env: ArtifactsTestEnvironment = ArtifactsTestEnvironment(tmp_path, test_type) + test_env.write_config_file(artifact_prebuild_clean=True, is_report_artifact=is_report_artifact) + test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name) + test_env.run() + test_env.check_artifact_present(test_env.artifact_path) + + +@pytest.mark.parametrize("test_type", ["main", "nonci"]) +@pytest.mark.parametrize("is_report_artifact", [True, False]) +def test_artifact_in_sources_no_prebuild_clean(tmp_path: pathlib.Path, + stdout_checker: FuzzyCallChecker, + is_report_artifact: bool, + test_type: str) -> None: + test_env: ArtifactsTestEnvironment = ArtifactsTestEnvironment(tmp_path, test_type) + test_env.write_config_file(artifact_prebuild_clean=False, is_report_artifact=is_report_artifact) + test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name) + if test_type == "main": + test_env.run(expect_failure=True) + stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) + test_env.check_artifact_absent() + elif test_type == "nonci": + test_env.run() + test_env.check_artifact_present(test_env.artifact_path) + else: + pytest.fail(f"Unexpected type type: {test_type}") + + +@pytest.mark.parametrize("no_archive", [False, True]) +def test_dir_artifact_in_sources_prebuild_clean(test_env: ArtifactsTestEnvironment, + no_archive: bool) -> None: + test_env.write_config_file(artifact_prebuild_clean=True) + test_env.create_artifacts_dir(test_env.src_dir) + test_env.settings.ArtifactCollector.no_archive = no_archive + test_env.run() + if no_archive: + test_env.check_artifact_present(test_env.artifact_in_dir) + else: + test_env.check_dir_zip_artifact_present() + + +def test_dir_artifact_in_sources_no_prebuild_clean(test_env: ArtifactsTestEnvironment, + stdout_checker: FuzzyCallChecker) -> None: + test_env.write_config_file(artifact_prebuild_clean=False) + test_env.create_artifacts_dir(test_env.src_dir) + test_env.run(expect_failure=True) + stdout_checker.assert_has_calls_with_param("already exist in '/.*' directory", is_regexp=True) + test_env.check_dir_zip_artifact_absent() + + +@pytest.mark.parametrize("is_zip", [True, False]) +@pytest.mark.parametrize("is_dir", [True, False]) +@pytest.mark.parametrize("prebuild_clean", [True, False]) +def test_artifact_in_artifacts_dir(test_env: ArtifactsTestEnvironment, + stdout_checker: FuzzyCallChecker, + is_zip: bool, + is_dir: bool, + prebuild_clean: bool) -> None: + test_env.write_config_file(artifact_prebuild_clean=prebuild_clean) + artifact_name: str = test_env.dir_name if is_dir else test_env.artifact_name + test_env.create_artifact_file(test_env.artifact_dir, artifact_name, is_zip) + test_env.run(expect_failure=True) + stdout_checker.assert_has_calls_with_param("already present in artifact directory") + + +def test_zip_artifact_no_archive(test_env: ArtifactsTestEnvironment) -> None: + test_env.settings.ArtifactCollector.no_archive = True + test_env.write_config_file(artifact_prebuild_clean=True) + test_env.create_artifact_file(test_env.artifact_dir, test_env.dir_name, is_zip=True) + test_env.run() + test_env.check_artifact_present(test_env.artifact_in_dir) + + +def test_wildcard(test_env: ArtifactsTestEnvironment) -> None: + test_env.write_config_file_wildcard(artifact_prebuild_clean=True) + test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name) + test_env.create_artifact_file(test_env.src_dir, test_env.artifact_name_with_suffix) + test_env.run() + test_env.check_artifact_present(test_env.artifact_path) + test_env.check_artifact_present(test_env.artifact_path_with_suffix) diff --git a/tests/test_regression.py b/tests/test_regression.py index 2740f585..0cf43304 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -1,6 +1,6 @@ # pylint: disable = redefined-outer-name -import py +import pathlib import pytest import P4 @@ -13,7 +13,7 @@ def test_which_universum_is_tested(docker_main: UniversumRunner, pytestconfig): # THIS TEST PATCHES ACTUAL SOURCES! Discretion is advised - init_file = pytestconfig.rootpath.joinpath("universum", "__init__.py") + init_file = pytestconfig.rootpath / "universum" / "__init__.py" backup = init_file.read_bytes() test_line = utils.randomize_name("THIS IS A TESTING VERSION") init_file.write_text(f"""__title__ = "Universum" @@ -42,14 +42,14 @@ def test_teardown_fixture_output_verification(print_text_on_teardown: None): @pytest.mark.parametrize("should_not_execute", [True, False], ids=['no-sources', 'deleted-sources']) -def test_clean_sources_exception(tmpdir: py.path.local, stdout_checker: FuzzyCallChecker, should_not_execute): - env = LocalTestEnvironment(tmpdir, "main") +def test_clean_sources_exception(tmp_path: pathlib.Path, stdout_checker: FuzzyCallChecker, should_not_execute): + env = LocalTestEnvironment(tmp_path, "main") env.settings.Vcs.type = "none" - source_directory = tmpdir + source_directory = tmp_path if should_not_execute: source_directory = source_directory / 'nonexisting_dir' env.settings.LocalMainVcs.source_dir = str(source_directory) - env.configs_file.write(f""" + env.configs_file.write_text(f""" from universum.configuration_support import Configuration configs = Configuration([dict(name="Test configuration", @@ -84,8 +84,8 @@ def test_non_utf8_environment(docker_main: UniversumRunner): @pytest.fixture() -def perforce_environment(perforce_workspace: PerforceWorkspace, tmpdir: py.path.local): - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="main") +def perforce_environment(perforce_workspace: PerforceWorkspace, tmp_path: pathlib.Path): + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="main") def test_p4_multiple_spaces_in_mappings(perforce_environment: P4TestEnvironment): @@ -102,7 +102,7 @@ def test_p4_repository_difference_format(perforce_environment: P4TestEnvironment """ perforce_environment.shelve_config(config) perforce_environment.run() - diff = perforce_environment.artifact_dir.join('REPOSITORY_DIFFERENCE.txt').read() + diff = (perforce_environment.artifact_dir / 'REPOSITORY_DIFFERENCE.txt').read_text() assert "This is a changed step name" in diff assert "b'" not in diff @@ -134,7 +134,7 @@ def test_p4_api_failed_opened(perforce_environment: P4TestEnvironment, mock_open perforce_environment.settings.Launcher.output = "file" perforce_environment.run() - log = perforce_environment.artifact_dir.join(f'{step_name}_log.txt').read() + log = (perforce_environment.artifact_dir / f'{step_name}_log.txt').read_text() assert "Module sh got exit code 1" in log assert "Getting file diff failed due to Perforce server internal error" in log @@ -168,7 +168,7 @@ def perforce_environment_with_files(perforce_environment: P4TestEnvironment): yield {"env": perforce_environment, "files": files} for entry in files: - perforce_environment.vcs_client.delete_file(entry.basename) + perforce_environment.vcs_client.delete_file(entry.name) def test_success_p4_resolve_unshelved(perforce_environment_with_files: dict, stdout_checker: FuzzyCallChecker): @@ -177,7 +177,7 @@ def test_success_p4_resolve_unshelved(perforce_environment_with_files: dict, std config = f""" from universum.configuration_support import Step, Configuration -configs = Configuration([Step(name="Print file", command=["bash", "-c", "cat '{p4_file.basename}'"])]) +configs = Configuration([Step(name="Print file", command=["bash", "-c", "cat '{p4_file.name}'"])]) """ env.shelve_config(config) cls = [env.vcs_client.shelve_file(p4_file, "This is changed line 1\nThis is unchanged line 2"), @@ -195,7 +195,7 @@ def test_fail_p4_resolve_unshelved(perforce_environment_with_files: dict, stdout config = f""" from universum.configuration_support import Step, Configuration -configs = Configuration([Step(name="Print file", command=["bash", "-c", "cat '{p4_file.basename}'"])]) +configs = Configuration([Step(name="Print file", command=["bash", "-c", "cat '{p4_file.name}'"])]) """ env.shelve_config(config) cls = [env.vcs_client.shelve_file(p4_file, "This is changed line 1\nThis is unchanged line 2"), @@ -204,7 +204,7 @@ def test_fail_p4_resolve_unshelved(perforce_environment_with_files: dict, stdout env.run(expect_failure=True) stdout_checker.assert_has_calls_with_param("Problems during merge while resolving shelved CLs") - stdout_checker.assert_has_calls_with_param(str(p4_file.basename)) + stdout_checker.assert_has_calls_with_param(str(p4_file.name)) def test_success_p4_resolve_unshelved_multiple(perforce_environment_with_files: dict): @@ -223,6 +223,26 @@ def test_success_p4_resolve_unshelved_multiple(perforce_environment_with_files: env.settings.PerforceMainVcs.shelve_cls.extend([cl_1, cl_2]) env.run() - repo_state = env.artifact_dir.join('REPOSITORY_STATE.txt').read() - assert p4_files[0].basename in repo_state - assert p4_files[1].basename in repo_state + repo_state = (env.artifact_dir / 'REPOSITORY_STATE.txt').read_text() + assert p4_files[0].name in repo_state + assert p4_files[1].name in repo_state + + +def test_exit_code_failed_report(perforce_environment: P4TestEnvironment): + """ + This test checks for previous bug where exceptions during result reporting led to exit code 0 + even when '--fail-unsuccessful' option was enabled (and some steps failed) + """ + config = """ +from universum.configuration_support import Configuration + +configs = Configuration([dict(name="Unsuccessful step", command=["exit", "1"])]) +""" + perforce_environment.shelve_config(config) + perforce_environment.settings.MainVcs.report_to_review = True + perforce_environment.settings.Swarm.server_url = "some_server" + perforce_environment.settings.Swarm.review_id = "some_id" + perforce_environment.settings.Swarm.change = perforce_environment.settings.PerforceMainVcs.shelve_cls[0] + perforce_environment.settings.Main.fail_unsuccessful = True + + perforce_environment.run(expect_failure=True) diff --git a/tests/test_report.py b/tests/test_report.py index 6ae9dab6..3b3da310 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,7 +1,7 @@ # pylint: disable = redefined-outer-name, abstract-method +import pathlib import pytest -import py from universum.modules.vcs.github_vcs import GithubToken from . import utils @@ -9,7 +9,7 @@ class ReportEnvironment(utils.BaseTestEnvironment): - def __init__(self, client: GitClient, directory: py.path.local): + def __init__(self, client: GitClient, directory: pathlib.Path): super().__init__(client, directory, "main", "") self.settings.Vcs.type = "github" @@ -30,8 +30,8 @@ def __init__(self, client: GitClient, directory: py.path.local): @pytest.fixture() -def report_environment(git_client: GitClient, tmpdir: py.path.local): - yield ReportEnvironment(git_client, tmpdir) +def report_environment(git_client: GitClient, tmp_path: pathlib.Path): + yield ReportEnvironment(git_client, tmp_path) def test_github_run(report_environment: ReportEnvironment, monkeypatch): diff --git a/tests/test_run_steps_filter.py b/tests/test_run_steps_filter.py index 4eadb979..86232efd 100644 --- a/tests/test_run_steps_filter.py +++ b/tests/test_run_steps_filter.py @@ -40,12 +40,12 @@ def step(name, cmd=False): @pytest.mark.parametrize("test_type", test_types) @pytest.mark.parametrize("filters, expected_logs, unexpected_logs", filters_parametrize_values) -def test_steps_filter(tmpdir, stdout_checker, filters, expected_logs, unexpected_logs, test_type): - params = get_cli_params(test_type, tmpdir) +def test_steps_filter(tmp_path, stdout_checker, filters, expected_logs, unexpected_logs, test_type): + params = get_cli_params(test_type, tmp_path) params.extend(["-o", "console"]) for _filter in filters: params.append(f"-f={_filter}") - params.extend(["-cfg", get_config_file_path(tmpdir, config)]) + params.extend(["-cfg", get_config_file_path(tmp_path, config)]) return_code = __main__.main(params) @@ -57,39 +57,39 @@ def test_steps_filter(tmpdir, stdout_checker, filters, expected_logs, unexpected @pytest.mark.parametrize("test_type", test_types) -def test_steps_filter_no_match(tmpdir, stdout_checker, test_type): +def test_steps_filter_no_match(tmp_path, stdout_checker, test_type): include_pattern = "asdf" exclude_pattern = "qwer" - cli_params = get_cli_params(test_type, tmpdir) + cli_params = get_cli_params(test_type, tmp_path) cli_params.extend(["-f", f"{include_pattern}:!{exclude_pattern}"]) - check_empty_config_error(tmpdir, stdout_checker, cli_params) + check_empty_config_error(tmp_path, stdout_checker, cli_params) stdout_checker.assert_has_calls_with_param(include_pattern) stdout_checker.assert_has_calls_with_param(exclude_pattern) @pytest.mark.parametrize("test_type", test_types) -def test_config_empty(tmpdir, stdout_checker, test_type): - check_empty_config_error(tmpdir, stdout_checker, get_cli_params(test_type, tmpdir)) +def test_config_empty(tmp_path, stdout_checker, test_type): + check_empty_config_error(tmp_path, stdout_checker, get_cli_params(test_type, tmp_path)) -def check_empty_config_error(tmpdir, stdout_checker, cli_params): - cli_params.extend(["-cfg", get_config_file_path(tmpdir, empty_config)]) +def check_empty_config_error(tmp_path, stdout_checker, cli_params): + cli_params.extend(["-cfg", get_config_file_path(tmp_path, empty_config)]) return_code = __main__.main(cli_params) assert return_code == 1 stdout_checker.assert_has_calls_with_param("Project configs are empty") -def get_config_file_path(tmpdir, text): - config_file = tmpdir.join("configs.py") +def get_config_file_path(tmp_path, text): + config_file = tmp_path / "configs.py" config_file.write_text(text, "utf-8") return str(config_file) -def get_cli_params(test_type, tmpdir): +def get_cli_params(test_type, tmp_path): if test_type == "nonci": return ["nonci"] return ["-vt", "none", - "-fsd", str(tmpdir), + "-fsd", str(tmp_path), "--clean-build"] diff --git a/tests/test_submit.py b/tests/test_submit.py index d6532ce6..432f8d10 100644 --- a/tests/test_submit.py +++ b/tests/test_submit.py @@ -3,7 +3,7 @@ import os from typing import Callable, Union import shutil -import py +import pathlib import pytest from . import utils @@ -13,16 +13,18 @@ @pytest.fixture() -def p4_submit_environment(perforce_workspace: PerforceWorkspace, tmpdir: py.path.local): - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="submit") +def p4_submit_environment(perforce_workspace: PerforceWorkspace, tmp_path: pathlib.Path): + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="submit") @pytest.mark.parametrize("branch", ["write-protected", "trigger-protected"]) def test_p4_error_forbidden_branch(p4_submit_environment: P4TestEnvironment, branch: str): - protected_dir = p4_submit_environment.vcs_client.root_directory.mkdir(branch) - file_to_add = protected_dir.join(utils.randomize_name("new_file") + ".txt") + protected_dir = p4_submit_environment.vcs_client.root_directory / branch + protected_dir.mkdir() + file_name = utils.randomize_name("new_file") + ".txt" + file_to_add = protected_dir / file_name text = "This is a new line in the file" - file_to_add.write(text + "\n") + file_to_add.write_text(text + "\n") p4_submit_environment.settings.Submit.reconcile_list = str(file_to_add) @@ -41,17 +43,17 @@ def test_p4_success_files_in_default(p4_submit_environment: P4TestEnvironment): p4_file = p4_submit_environment.vcs_client.repo_file p4.run_edit(str(p4_file)) text = "This text should be in file" - p4_file.write(text + "\n") + p4_file.write_text(text + "\n") # This file should be successfully submitted file_name = utils.randomize_name("new_file") + ".txt" - new_file = p4_submit_environment.vcs_client.root_directory.join(file_name) - new_file.write("This is a new file" + "\n") + new_file = p4_submit_environment.vcs_client.root_directory / file_name + new_file.write_text("This is a new file" + "\n") p4_submit_environment.settings.Submit.reconcile_list = str(new_file) p4_submit_environment.run() - assert text in p4_file.read() + assert text in p4_file.read_text() def test_p4_error_files_in_default_and_reverted(p4_submit_environment: P4TestEnvironment): @@ -60,19 +62,21 @@ def test_p4_error_files_in_default_and_reverted(p4_submit_environment: P4TestEnv p4_file = p4_submit_environment.vcs_client.repo_file p4.run_edit(str(p4_file)) text_default = "This text should be in file" - p4_file.write(text_default + "\n") + p4_file.write_text(text_default + "\n") # This file must fail submit and remain unchanged while not checked out any more - protected_dir = p4_submit_environment.vcs_client.root_directory.mkdir("write-protected") - new_file = protected_dir.join(utils.randomize_name("new_file") + ".txt") + protected_dir = p4_submit_environment.vcs_client.root_directory / "write-protected" + protected_dir.mkdir() + file_name = utils.randomize_name("new_file") + ".txt" + new_file = protected_dir / file_name text_new = "This is a new line in the file" - new_file.write(text_new + "\n") + new_file.write_text(text_new + "\n") p4_submit_environment.settings.Submit.reconcile_list = str(new_file) p4_submit_environment.run(expect_failure=True) - assert text_default in p4_file.read() - assert text_new in new_file.read() + assert text_default in p4_file.read_text() + assert text_new in new_file.read_text() class SubmitterParameters: @@ -110,11 +114,11 @@ def inner(environment): @pytest.fixture(params=["git", "p4"]) -def submit_environment(request, perforce_workspace: PerforceWorkspace, git_client: GitClient, tmpdir: py.path.local): +def submit_environment(request, perforce_workspace: PerforceWorkspace, git_client: GitClient, tmp_path: pathlib.Path): if request.param == "git": - yield GitTestEnvironment(git_client, tmpdir, test_type="submit") + yield GitTestEnvironment(git_client, tmp_path, test_type="submit") else: - yield P4TestEnvironment(perforce_workspace, tmpdir, test_type="submit") + yield P4TestEnvironment(perforce_workspace, tmp_path, test_type="submit") def test_error_no_repo(submit_environment: Union[GitTestEnvironment, P4TestEnvironment], stdout_checker: FuzzyCallChecker): @@ -138,22 +142,22 @@ def test_success_commit_add_modify_remove_one_file(submit_parameters: Callable, parameters = submit_parameters(submit_environment) file_name = utils.randomize_name("new_file") + ".txt" - temp_file = parameters.environment.vcs_client.root_directory.join(file_name) + temp_file = parameters.environment.vcs_client.root_directory / file_name file_path = str(temp_file) # Add a file - temp_file.write("This is a new file" + "\n") + temp_file.write_text("This is a new file" + "\n") parameters.assert_submit_success([file_path]) assert parameters.file_present(file_path) # Modify a file text = "This is a new line in the file" - temp_file.write(text + "\n") + temp_file.write_text(text + "\n") parameters.assert_submit_success([file_path]) assert parameters.text_in_file(text, file_path) # Delete a file - temp_file.remove() + temp_file.unlink() parameters.assert_submit_success([file_path]) assert not parameters.file_present(file_path) @@ -163,8 +167,8 @@ def test_success_ignore_new_and_deleted_while_edit_only(submit_parameters: Calla parameters = submit_parameters(submit_environment) new_file_name = utils.randomize_name("new_file") + ".txt" - temp_file = parameters.environment.vcs_client.root_directory.join(new_file_name) - temp_file.write("This is a new temp file" + "\n") + temp_file = parameters.environment.vcs_client.root_directory / new_file_name + temp_file.write_text("This is a new temp file" + "\n") deleted_file_path = str(parameters.environment.vcs_client.repo_file) deleted_file_name = os.path.basename(deleted_file_path) os.remove(deleted_file_path) @@ -184,7 +188,7 @@ def test_success_commit_modified_while_edit_only(submit_parameters: Callable, target_file = parameters.environment.vcs_client.repo_file text = utils.randomize_name("This is change ") - target_file.write(text + "\n") + target_file.write_text(text + "\n") parameters.assert_submit_success([str(target_file)], edit_only=True) assert parameters.text_in_file(text, str(target_file)) @@ -194,7 +198,7 @@ def test_error_review(submit_parameters: Callable, submit_environment: Union[Git parameters = submit_parameters(submit_environment) target_file = parameters.environment.vcs_client.repo_file - target_file.write("This is some change") + target_file.write_text("This is some change") parameters.submit_path_list([str(target_file)], review=True, expect_failure=True) parameters.stdout_checker.assert_has_calls_with_param("not supported") @@ -207,21 +211,23 @@ def test_success_reconcile_directory(submit_parameters: Callable, dir_name = utils.randomize_name("new_directory") # Create and reconcile new directory - tmp_dir = parameters.environment.vcs_client.root_directory.mkdir(dir_name) + tmp_dir = parameters.environment.vcs_client.root_directory / dir_name + tmp_dir.mkdir() for i in range(0, 9): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write("This is some file" + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text("This is some file" + "\n") parameters.assert_submit_success([str(tmp_dir) + "/"]) for i in range(0, 9): - file_path = tmp_dir.join(f"new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert parameters.file_present(str(file_path)) # Create and reconcile a directory in a directory - another_dir = tmp_dir.mkdir("another_directory") - tmp_file = another_dir.join("new_file.txt") - tmp_file.write("This is some file" + "\n") + another_dir = tmp_dir / "another_directory" + another_dir.mkdir() + tmp_file = another_dir / "new_file.txt" + tmp_file.write_text("This is some file" + "\n") parameters.assert_submit_success([str(tmp_dir) + "/"]) assert parameters.file_present(str(tmp_file)) @@ -229,13 +235,13 @@ def test_success_reconcile_directory(submit_parameters: Callable, # Modify some vcs text = utils.randomize_name("This is change ") for i in range(0, 9, 2): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") parameters.assert_submit_success([str(tmp_dir) + "/"], edit_only=True) for i in range(0, 9, 2): - file_path = tmp_dir.join(f"/new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert parameters.text_in_file(text, str(file_path)) parameters.environment.settings.Submit.edit_only = False @@ -252,106 +258,109 @@ def test_success_reconcile_wildcard(submit_parameters: Callable, dir_name = utils.randomize_name("new_directory") # Create embedded directories, partially reconcile - tmp_dir = parameters.environment.vcs_client.root_directory.mkdir(dir_name) - inner_dir = tmp_dir.mkdir("inner_directory") + tmp_dir = parameters.environment.vcs_client.root_directory / dir_name + tmp_dir.mkdir() + inner_dir = tmp_dir / "inner_directory" + inner_dir.mkdir() text = "This is some file" + "\n" for i in range(0, 9): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write(text) - tmp_file = tmp_dir.join(f"another_file{i}.txt") - tmp_file.write(text) - tmp_file = inner_dir.join(f"new_file{i}.txt") - tmp_file.write(text) + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text(text) + tmp_file = tmp_dir / f"another_file{i}.txt" + tmp_file.write_text(text) + tmp_file = inner_dir / f"new_file{i}.txt" + tmp_file.write_text(text) parameters.assert_submit_success([str(tmp_dir) + "/new_file*.txt"]) for i in range(0, 9): file_name = f"new_file{i}.txt" - file_path = tmp_dir.join(file_name) + file_path = tmp_dir / file_name assert parameters.file_present(str(file_path)) - file_path = inner_dir.join(file_name) + file_path = inner_dir / file_name assert not parameters.file_present(str(file_path)) file_name = f"another_file{i}.txt" - file_path = tmp_dir.join(file_name) + file_path = tmp_dir / file_name assert not parameters.file_present(str(file_path)) # Create one more directory other_dir_name = utils.randomize_name("new_directory") - other_tmp_dir = parameters.environment.vcs_client.root_directory.mkdir(other_dir_name) + other_tmp_dir = parameters.environment.vcs_client.root_directory / other_dir_name + other_tmp_dir.mkdir() for i in range(0, 9): - tmp_file = other_tmp_dir.join(f"new_file{i}.txt") - tmp_file.write("This is some file" + "\n") + tmp_file = other_tmp_dir / f"new_file{i}.txt" + tmp_file.write_text("This is some file" + "\n") parameters.assert_submit_success([str(parameters.environment.vcs_client.root_directory) + "/new_directory*/"]) for i in range(0, 9): file_name = f"new_file{i}.txt" - file_path = other_tmp_dir.join(file_name) + file_path = other_tmp_dir / file_name assert parameters.file_present(str(file_path)) - file_path = inner_dir.join(file_name) + file_path = inner_dir / file_name assert parameters.file_present(str(file_path)) file_name = f"another_file{i}.txt" - file_path = tmp_dir.join(file_name) + file_path = tmp_dir / file_name assert parameters.file_present(str(file_path)) # Modify some vcs text = utils.randomize_name("This is change ") for i in range(0, 9, 2): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = inner_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = tmp_dir.join(f"another_file{i}.txt") - tmp_file.write(text + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = inner_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = tmp_dir / f"another_file{i}.txt" + tmp_file.write_text(text + "\n") parameters.assert_submit_success([str(tmp_dir) + "/new_file*.txt"], edit_only=True) for i in range(0, 9, 2): - file_path = tmp_dir.join(f"/new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert parameters.text_in_file(text, str(file_path)) - file_path = inner_dir.join(f"/new_file{i}.txt") + file_path = inner_dir / f"new_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) - file_path = tmp_dir.join(f"/another_file{i}.txt") + file_path = tmp_dir / f"another_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) # Test subdirectory wildcard text = utils.randomize_name("This is change ") for i in range(1, 9, 2): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = inner_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = tmp_dir.join(f"another_file{i}.txt") - tmp_file.write(text + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = inner_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = tmp_dir / f"another_file{i}.txt" + tmp_file.write_text(text + "\n") parameters.assert_submit_success([str(tmp_dir) + "/*/*.txt"]) for i in range(1, 9, 2): - file_path = inner_dir.join(f"new_file{i}.txt") + file_path = inner_dir / f"new_file{i}.txt" assert parameters.text_in_file(text, str(file_path)) - file_path = tmp_dir.join(f"new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) - file_path = tmp_dir.join(f"another_file{i}.txt") + file_path = tmp_dir / f"another_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) # Test edit-only subdirectory wildcard text = utils.randomize_name("This is change ") for i in range(0, 9, 3): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = inner_dir.join(f"new_file{i}.txt") - tmp_file.write(text + "\n") - tmp_file = tmp_dir.join("another_file{i}.txt") - tmp_file.write(text + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = inner_dir / f"new_file{i}.txt" + tmp_file.write_text(text + "\n") + tmp_file = tmp_dir / "another_file{i}.txt" + tmp_file.write_text(text + "\n") parameters.assert_submit_success([str(tmp_dir) + "/*/*.txt"], edit_only=True) for i in range(0, 9, 3): - file_path = inner_dir.join(f"new_file{i}.txt") + file_path = inner_dir / f"new_file{i}.txt" assert parameters.text_in_file(text, str(file_path)) - file_path = tmp_dir.join(f"new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) - file_path = tmp_dir.join(f"another_file{i}.txt") + file_path = tmp_dir / f"another_file{i}.txt" assert not parameters.text_in_file(text, str(file_path)) parameters.environment.settings.Submit.edit_only = False @@ -370,21 +379,22 @@ def test_success_reconcile_partial(submit_parameters: Callable, parameters = submit_parameters(submit_environment) dir_name = utils.randomize_name("new_directory") - tmp_dir = parameters.environment.vcs_client.root_directory.mkdir(dir_name) + tmp_dir = parameters.environment.vcs_client.root_directory / dir_name + tmp_dir.mkdir() for i in range(0, 9): - tmp_file = tmp_dir.join(f"new_file{i}.txt") - tmp_file.write("This is some file" + "\n") + tmp_file = tmp_dir / f"new_file{i}.txt" + tmp_file.write_text("This is some file" + "\n") - reconcile_list = [str(tmp_dir.join(f"new_file{i}.txt")) for i in range(0, 4)] + reconcile_list = [str(tmp_dir / f"new_file{i}.txt") for i in range(0, 4)] reconcile_list.extend(["", " ", "\n"]) parameters.assert_submit_success(reconcile_list) for i in range(0, 4): - file_path = tmp_dir.join(f"new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert parameters.file_present(str(file_path)) for i in range(5, 9): - file_path = tmp_dir.join(f"new_file{i}.txt") + file_path = tmp_dir / f"new_file{i}.txt" assert not parameters.file_present(str(file_path)) # Delete a directory diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 1bb00343..f047ea38 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,7 +1,7 @@ # pylint: disable = redefined-outer-name +import pathlib import git -import py import pytest from . import utils @@ -10,24 +10,28 @@ @pytest.fixture -def unicode_dir(tmpdir: py.path.local): - yield tmpdir.mkdir("Юніко́д з пробелами") +def unicode_dir(tmp_path: pathlib.Path): + unicode_dir_path = tmp_path / "Юніко́д з пробелами" + unicode_dir_path.mkdir() + yield unicode_dir_path @pytest.mark.parametrize("vcs", ["git", "p4"]) @pytest.mark.parametrize("test_type", ["main", "poll", "submit"]) -def test_unicode(vcs, test_type, perforce_workspace: PerforceWorkspace, git_client: GitClient, unicode_dir: py.path.local): +def test_unicode(vcs, test_type, perforce_workspace: PerforceWorkspace, git_client: GitClient, unicode_dir: pathlib.Path): env: utils.BaseTestEnvironment if vcs == "git": # change git client root dir to unicode path - work_dir = unicode_dir.mkdir("client") + work_dir = unicode_dir / "client" + work_dir.mkdir() git_client.repo = git.Repo.clone_from(git_client.server.url, work_dir) git_client.root_directory = work_dir env = GitTestEnvironment(git_client, unicode_dir, test_type=test_type) elif vcs == "p4": # change workspace root dir to unicode path - root = unicode_dir.mkdir("workspace") + root = unicode_dir / "workspace" + root.mkdir() client = perforce_workspace.p4.fetch_client(perforce_workspace.client_name) client["Root"] = str(root) perforce_workspace.root_directory = root @@ -39,16 +43,18 @@ def test_unicode(vcs, test_type, perforce_workspace: PerforceWorkspace, git_clie assert False, "Unsupported vcs type" if test_type == "submit": - temp_file = env.vcs_client.root_directory.join(utils.randomize_name("new_file") + ".txt") - temp_file.write("This is a new file" + "\n") + file_name = utils.randomize_name("new_file") + ".txt" + temp_file = env.vcs_client.root_directory / file_name + temp_file.write_text("This is a new file" + "\n") env.settings.Submit.reconcile_list = str(temp_file) env.run() -def test_unicode_main_local_vcs(unicode_dir: py.path.local): - work_dir = unicode_dir.mkdir("local_sources") - work_dir.join("source_file").write("Source file contents") +def test_unicode_main_local_vcs(unicode_dir: pathlib.Path): + work_dir = unicode_dir / "local_sources" + work_dir.mkdir() + (work_dir / "source_file").write_text("Source file contents") env = utils.LocalTestEnvironment(unicode_dir, "main") env.settings.Vcs.type = "none" diff --git a/tests/utils.py b/tests/utils.py index 1fcc08ab..47f2f930 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,11 +5,11 @@ import socket import string import sys +import pathlib from typing import Type from docker.models.containers import Container import httpretty -import py import pytest from universum import submit, poll, main, github_handler, nonci, __main__ @@ -109,20 +109,20 @@ def create_empty_settings(test_type: str) -> ModuleNamespace: class BaseVcsClient: - def __init__(self): - self.root_directory: py.path.local - self.repo_file: py.path.local + def __init__(self) -> None: + self.root_directory: pathlib.Path + self.repo_file: pathlib.Path def get_last_change(self): pass - def file_present(self, file_path: str) -> bool: + def file_present(self, file_path: str) -> bool: # type: ignore pass - def text_in_file(self, text: str, file_path: str) -> bool: + def text_in_file(self, text: str, file_path: str) -> bool: # type: ignore pass - def make_a_change(self) -> str: + def make_a_change(self) -> str: # type: ignore pass @@ -191,8 +191,8 @@ def assert_request_body_contained(key, value): class BaseTestEnvironment: - def __init__(self, client: BaseVcsClient, directory: py.path.local, test_type: str, db_file: str): - self.temp_dir: py.path.local = directory + def __init__(self, client: BaseVcsClient, directory: pathlib.Path, test_type: str, db_file: str): + self.temp_dir: pathlib.Path = directory self.vcs_client = client self.settings: ModuleNamespace = create_empty_settings(test_type) @@ -200,22 +200,26 @@ def __init__(self, client: BaseVcsClient, directory: py.path.local, test_type: s self.settings.Poll.db_file = db_file self.settings.JenkinsServerForTrigger.trigger_url = "https://localhost/?cl=%s" self.settings.AutomationServer.type = "jenkins" - self.settings.ProjectDirectory.project_root = str(self.temp_dir.mkdir("project_root")) + project_root_dir = self.temp_dir / "project_root" + project_root_dir.mkdir() + self.settings.ProjectDirectory.project_root = str(project_root_dir) elif test_type == "submit": self.settings.Submit.commit_message = "Test CL" # For submitter, the main working dir (project_root) should be the root # of the VCS workspace/client self.settings.ProjectDirectory.project_root = str(self.vcs_client.root_directory) elif test_type in ("main", "nonci"): - self.configs_file = self.temp_dir.join("configs.py") - self.configs_file.write(simple_test_config) + self.configs_file = self.temp_dir / "configs.py" + self.configs_file.write_text(simple_test_config) self.settings.Launcher.config_path = str(self.configs_file) - self.artifact_dir = self.temp_dir.mkdir("artifacts") + self.artifact_dir = self.temp_dir / "artifacts" + self.artifact_dir.mkdir() self.settings.ArtifactCollector.artifact_dir = str(self.artifact_dir) # The project_root directory must not exist before launching main - self.settings.ProjectDirectory.project_root = str(self.temp_dir.join("project_root")) + self.settings.ProjectDirectory.project_root = str(self.temp_dir / "project_root") if test_type == "nonci": - self.temp_dir.mkdir("project_root") + self.nonci_dir = self.temp_dir / "project_root" + self.nonci_dir.mkdir() self.settings.Launcher.output = "console" self.settings.AutomationServer.type = "local" self.settings.Output.type = "term" @@ -257,4 +261,8 @@ def __init__(self, directory, test_type): if test_type != "nonci": self.settings.Vcs.type = "none" if test_type == "main": - self.settings.LocalMainVcs.source_dir = str(directory.ensure_dir('project_sources')) + self.src_dir = directory / 'project_sources' + self.src_dir.mkdir() + self.settings.LocalMainVcs.source_dir = str(self.src_dir) + if test_type == "nonci": + self.src_dir = self.nonci_dir diff --git a/universum/__init__.py b/universum/__init__.py index fe70ce34..97c18b90 100644 --- a/universum/__init__.py +++ b/universum/__init__.py @@ -1,2 +1,2 @@ __title__ = "Universum" -__version__ = "0.19.14" +__version__ = "0.19.15" diff --git a/universum/analyzers/clang_format.py b/universum/analyzers/clang_format.py new file mode 100755 index 00000000..56a64f80 --- /dev/null +++ b/universum/analyzers/clang_format.py @@ -0,0 +1,67 @@ +import argparse +import pathlib +from typing import Callable, List, Optional, Tuple + +import yaml + +from . import utils, diff_utils + + +def clang_format_argument_parser() -> argparse.ArgumentParser: + parser = diff_utils.diff_analyzer_argument_parser("Clang-format analyzer", __file__, "clang-format") + parser.add_argument("--executable", "-e", dest="executable", default="clang-format", + help="The name of the clang-format executable, default: clang-format") + parser.add_argument("--style", dest="style", + help="The 'style' parameter of the clang-format. Can be literal 'file' string or " + "path to real file. See the clang-format documentation for details.") + return parser + + +def _add_style_param_if_present(cmd: List[str], settings: argparse.Namespace) -> None: + if settings.style: + cmd.extend(["-style", settings.style]) + + +@utils.sys_exit +@utils.analyzer(clang_format_argument_parser()) +def main(settings: argparse.Namespace) -> List[utils.ReportData]: + settings.name = "clang-format" + diff_utils.diff_analyzer_common_main(settings) + + html_diff_file_writer: Optional[Callable[[pathlib.Path, List[str], List[str]], None]] = None + if settings.write_html: + wrapcolumn, tabsize = _get_wrapcolumn_tabsize(settings) + html_diff_file_writer = diff_utils.HtmlDiffFileWriter(settings.target_folder, wrapcolumn, tabsize) + + files: List[Tuple[pathlib.Path, pathlib.Path]] = [] + for src_file_absolute, target_file_absolute, _ in utils.get_files_with_absolute_paths(settings): + files.append((src_file_absolute, target_file_absolute)) + cmd = [settings.executable, src_file_absolute] + _add_style_param_if_present(cmd, settings) + output, _ = utils.run_for_output(cmd) + with open(target_file_absolute, "w", encoding="utf-8") as output_file: + output_file.write(output) + + return diff_utils.diff_analyzer_output_parser(files, html_diff_file_writer) + + +def _get_wrapcolumn_tabsize(settings: argparse.Namespace) -> Tuple[int, int]: + cmd = [settings.executable, "--dump-config"] + _add_style_param_if_present(cmd, settings) + output, error = utils.run_for_output(cmd) + if error: + raise utils.AnalyzerException(message="clang-format --dump-config failed with the following error output: " + + error) + try: + clang_style = yaml.safe_load(output) + return clang_style["ColumnLimit"], clang_style["IndentWidth"] + except yaml.YAMLError as parse_error: + raise utils.AnalyzerException(message="Parsing of clang-format config produced the following error: " + + str(parse_error)) + except KeyError as key_error: + raise utils.AnalyzerException(message="Cannot find key in yaml during parsing of clang-format config: " + + str(key_error)) + + +if __name__ == "__main__": + main() diff --git a/universum/analyzers/diff_utils.py b/universum/analyzers/diff_utils.py new file mode 100644 index 00000000..493cf7de --- /dev/null +++ b/universum/analyzers/diff_utils.py @@ -0,0 +1,133 @@ +import argparse +import difflib +import pathlib +import shutil +from typing import Callable, List, Optional, Tuple + +from . import utils + + +class HtmlDiffFileWriter: + + def __init__(self, target_folder: pathlib.Path, wrapcolumn: int, tabsize: int) -> None: + self.target_folder = target_folder + self.differ = difflib.HtmlDiff(tabsize=tabsize, wrapcolumn=wrapcolumn) + + def __call__(self, file: pathlib.Path, src: List[str], target: List[str]) -> None: + file_relative = file.relative_to(pathlib.Path.cwd()) + out_file_name: str = str(file_relative).replace('/', '_') + '.html' + with open(self.target_folder.joinpath(out_file_name), 'w', encoding="utf-8") as out_file: + out_file.write(self.differ.make_file(src, target, context=False)) + + +DiffWriter = Callable[[pathlib.Path, List[str], List[str]], None] + + +def diff_analyzer_output_parser(files: List[Tuple[pathlib.Path, pathlib.Path]], + write_diff_file: Optional[DiffWriter] + ) -> List[utils.ReportData]: + result: List[utils.ReportData] = [] + for src_file, dst_file in files: + with open(src_file, encoding="utf-8") as src: + src_lines = src.readlines() + with open(dst_file, encoding="utf-8") as fixed: + fixed_lines = fixed.readlines() + + issues = _get_issues_from_diff(src_file, src_lines, fixed_lines) + if issues and write_diff_file: + write_diff_file(src_file, src_lines, fixed_lines) + result.extend(issues) + return result + + +def _get_issues_from_diff(src_file: pathlib.Path, src: List[str], target: List[str]) -> List[utils.ReportData]: + result = [] + matching_blocks: List[difflib.Match] = \ + difflib.SequenceMatcher(a=src, b=target).get_matching_blocks() + previous_match = matching_blocks[0] + for match in matching_blocks[1:]: + block = _get_mismatching_block(previous_match, match, src, target) + previous_match = match + if not block: + continue + line, before, after = block + path: pathlib.Path = src_file.relative_to(pathlib.Path.cwd()) + message = _get_issue_message(before, after) + result.append(utils.ReportData( + symbol="Code Style issue", + message=message, + path=str(path), + line=line + )) + + return result + + +def _get_issue_message(before: str, after: str) -> str: + # The maximum number of lines to write separate comments for + # If exceeded, summarized comment will be provided instead + max_lines = 11 + diff_size = len(before.splitlines()) + if diff_size > max_lines: + message = f"\nLarge block of code ({diff_size} lines) has issues\n" + \ + f"Non-compliant code blocks exceeding {max_lines} lines are not reported\n" + else: + # Message with before&after + message = f"\nOriginal code:\n```diff\n{before}```\n" + \ + f"Fixed code:\n```diff\n{after}```\n" + return message + + +def _get_mismatching_block(previous_match: difflib.Match, # src[a:a+size] = target[b:b+size] + match: difflib.Match, + src: List[str], target: List[str] + ) -> Optional[Tuple[int, str, str]]: + previous_match_end_in_src = previous_match.a + previous_match.size + previous_match_end_in_target = previous_match.b + previous_match.size + match_start_in_src = match.a + match_start_in_target = match.b + if previous_match_end_in_src == match_start_in_src: + return None + line = match_start_in_src + before = _get_text_for_block(previous_match_end_in_src - 1, match_start_in_src, src) + after = _get_text_for_block(previous_match_end_in_target - 1, match_start_in_target, target) + return line, before, after + + +def _get_text_for_block(start: int, end: int, lines: List[str]) -> str: + return _replace_whitespace_characters(''.join(lines[start: end])) + + +_whitespace_character_mapping = { + " ": "\u00b7", + "\t": "\u2192\u2192\u2192\u2192", + "\n": "\u2193\u000a" +}.items() + + +def _replace_whitespace_characters(line: str) -> str: + for old_str, new_str in _whitespace_character_mapping: + line = line.replace(old_str, new_str) + return line + + +def diff_analyzer_argument_parser(description: str, module_path: str, output_directory: str) -> argparse.ArgumentParser: + parser = utils.create_parser(description, module_path) + parser.add_argument("--output-directory", "-od", dest="output_directory", default=output_directory, + help=f"Directory to store fixed files and HTML files with diff; the default " + f"value is '{output_directory}'. Has to be distinct from source directory") + parser.add_argument("--report-html", dest="write_html", action="store_true", default=False, + help="(optional) Set to generate html reports for each modified file") + return parser + + +def diff_analyzer_common_main(settings: argparse.Namespace) -> None: + settings.target_folder = utils.normalize_path(settings.output_directory) + if settings.target_folder.exists() and settings.target_folder.samefile(pathlib.Path.cwd()): + raise EnvironmentError("Target folder must not be identical to source folder") + + settings.target_folder.mkdir(parents=True, exist_ok=True) + + if not shutil.which(settings.executable): + raise EnvironmentError(f"{settings.name} executable '{settings.executable}' is not found. " + f"Please install {settings.name} or fix the executable name.") diff --git a/universum/analyzers/mypy.py b/universum/analyzers/mypy.py index d5c95e54..a8b0544a 100644 --- a/universum/analyzers/mypy.py +++ b/universum/analyzers/mypy.py @@ -1,12 +1,11 @@ import argparse - from typing import List from . import utils def mypy_argument_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Mypy analyzer") + parser = utils.create_parser("Mypy analyzer", __file__) parser.add_argument("--config-file", dest="config_file", type=str, help="Specify a configuration file.") utils.add_python_version_argument(parser) return parser diff --git a/universum/analyzers/pylint.py b/universum/analyzers/pylint.py index 7b383f89..11cb8a67 100644 --- a/universum/analyzers/pylint.py +++ b/universum/analyzers/pylint.py @@ -1,13 +1,12 @@ import argparse import json - from typing import List from . import utils def pylint_argument_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Pylint analyzer") + parser = utils.create_parser("Pylint analyzer", __file__) parser.add_argument("--rcfile", dest="rcfile", type=str, help="Specify a configuration file.") utils.add_python_version_argument(parser) return parser diff --git a/universum/analyzers/uncrustify.py b/universum/analyzers/uncrustify.py index 0d3cf3be..7afc548d 100755 --- a/universum/analyzers/uncrustify.py +++ b/universum/analyzers/uncrustify.py @@ -1,159 +1,61 @@ import argparse -import difflib import os -import shutil -from pathlib import Path - +import pathlib +import re from typing import Callable, List, Optional, Tuple -from . import utils +from . import utils, diff_utils def uncrustify_argument_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Uncrustify analyzer") + parser = diff_utils.diff_analyzer_argument_parser("Uncrustify analyzer", __file__, "uncrustify") parser.add_argument("--cfg-file", "-cf", dest="cfg_file", help="Name of the configuration file of Uncrustify; " "can also be set via 'UNCRUSTIFY_CONFIG' env. variable") - parser.add_argument("--output-directory", "-od", dest="output_directory", default="uncrustify", - help="Directory to store fixed files, generated by Uncrustify " - "and HTML files with diff; the default value is 'uncrustify'" - "Has to be distinct from source directory") - parser.add_argument("--report-html", dest="write_html", action="store_true", default=False, - help="(optional) Set to generate html reports for each modified file") return parser @utils.sys_exit @utils.analyzer(uncrustify_argument_parser()) def main(settings: argparse.Namespace) -> List[utils.ReportData]: - if not shutil.which('uncrustify'): - raise EnvironmentError("Please install uncrustify") + settings.name = "uncrustify" + settings.executable = "uncrustify" + diff_utils.diff_analyzer_common_main(settings) + if not settings.cfg_file and 'UNCRUSTIFY_CONFIG' not in os.environ: - raise EnvironmentError("Please specify the '--cfg_file' parameter " - "or set an env. variable 'UNCRUSTIFY_CONFIG'") - target_folder: Path = utils.normalize(settings.output_directory) - if target_folder.exists() and target_folder.samefile(Path.cwd()): - raise EnvironmentError("Target and source folders for uncrustify are not allowed to match") - html_diff_file_writer: Optional[Callable[[Path, List[str], List[str]], None]] = None + raise EnvironmentError("Please specify the '--cfg-file' parameter " + "or set 'UNCRUSTIFY_CONFIG' environment variable") + + html_diff_file_writer: Optional[Callable[[pathlib.Path, List[str], List[str]], None]] = None if settings.write_html: wrapcolumn, tabsize = _get_wrapcolumn_tabsize(settings.cfg_file) - html_diff_file_writer = HtmlDiffFileWriter(target_folder, wrapcolumn, tabsize) + html_diff_file_writer = diff_utils.HtmlDiffFileWriter(settings.target_folder, wrapcolumn, tabsize) - files: List[Tuple[Path, Path]] = [] - for src_file in settings.file_list: - src_file_absolute = utils.normalize(src_file) - src_file_relative = src_file_absolute.relative_to(Path.cwd()) - target_file_absolute: Path = target_folder.joinpath(src_file_relative) - files.append((src_file_absolute, target_file_absolute)) cmd = ["uncrustify", "-q", "-c", settings.cfg_file, "--prefix", settings.output_directory] - cmd.extend(settings.file_list) - utils.run_for_output(cmd) - return uncrustify_output_parser(files, html_diff_file_writer) - - -class HtmlDiffFileWriter: - - def __init__(self, target_folder: Path, wrapcolumn: int, tabsize: int) -> None: - self.target_folder = target_folder - self.differ = difflib.HtmlDiff(tabsize=tabsize, wrapcolumn=wrapcolumn) - - def __call__(self, file: Path, src: List[str], target: List[str]) -> None: - file_relative = file.relative_to(Path.cwd()) - out_file_name: str = str(file_relative).replace('/', '_') + '.html' - with open(self.target_folder.joinpath(out_file_name), 'w', encoding="utf-8") as out_file: - out_file.write(self.differ.make_file(src, target, context=False)) - - -def uncrustify_output_parser(files: List[Tuple[Path, Path]], - write_diff_file: Optional[Callable[[Path, List[str], List[str]], None]] - ) -> List[utils.ReportData]: - result: List[utils.ReportData] = [] - for src_file, uncrustify_file in files: - with open(src_file, encoding="utf-8") as src: - src_lines = src.readlines() - with open(uncrustify_file, encoding="utf-8") as fixed: - fixed_lines = fixed.readlines() + files: List[Tuple[pathlib.Path, pathlib.Path]] = [] + for src_file_absolute, target_file_absolute, src_file_relative in utils.get_files_with_absolute_paths(settings): + files.append((src_file_absolute, target_file_absolute)) + cmd.append(src_file_relative) - issues = _get_issues_from_diff(src_file, src_lines, fixed_lines) - if issues and write_diff_file: - write_diff_file(src_file, src_lines, fixed_lines) - result.extend(issues) - return result + utils.run_for_output(cmd) + return diff_utils.diff_analyzer_output_parser(files, html_diff_file_writer) def _get_wrapcolumn_tabsize(cfg_file: str) -> Tuple[int, int]: + wrapcolumn = 120 + tabsize = 4 with open(cfg_file, encoding="utf-8") as config: for line in config.readlines(): - if line.startswith("code_width"): - wrapcolumn = int(line.split()[2]) - if line.startswith("input_tab_size"): - tabsize = int(line.split()[2]) + match = re.match(r"^\s*([A-Za-z_]+)\s*[,\=]?\s*([0-9]+)\s*$", line) + if not match: + continue + groups = match.groups() + if groups[0] == "code_width": + wrapcolumn = int(groups[1]) + if groups[0] == "input_tab_size": + tabsize = int(groups[1]) return wrapcolumn, tabsize -def _get_issues_from_diff(src_file: Path, src: List[str], target: List[str]) -> List[utils.ReportData]: - result = [] - matching_blocks: List[difflib.Match] = \ - difflib.SequenceMatcher(a=src, b=target).get_matching_blocks() - previous_match = matching_blocks[0] - for match in matching_blocks[1:]: - block = _get_mismatching_block(previous_match, match, src, target) - previous_match = match - if not block: - continue - line, before, after = block - path: Path = src_file.relative_to(Path.cwd()) - message = _get_issue_message(before, after) - result.append(utils.ReportData( - symbol="Uncrustify Code Style issue", - message=message, - path=str(path), - line=line - )) - - return result - - -def _get_issue_message(before: str, after: str) -> str: - # The maximum number of lines to write separate comments for - # If exceeded, summarized comment will be provided instead - max_lines = 11 - diff_size = len(before.splitlines()) - if diff_size > max_lines: - message = f"\nLarge block of code ({diff_size} lines) has issues\n" + \ - f"Non-compliant code blocks exceeding {max_lines} lines are not reported\n" - else: - # Message with before&after - message = f"\nOriginal code:\n```diff\n{before}```\n" + \ - f"Uncrustify generated code:\n```diff\n{after}```\n" - return message - - -def _get_mismatching_block(previous_match: difflib.Match, # src[a:a+size] = target[b:b+size] - match: difflib.Match, - src: List[str], target: List[str] - ) -> Optional[Tuple[int, str, str]]: - previous_match_end_in_src = previous_match.a + previous_match.size - previous_match_end_in_target = previous_match.b + previous_match.size - match_start_in_src = match.a - match_start_in_target = match.b - if previous_match_end_in_src == match_start_in_src: - return None - line = match_start_in_src - before = _get_text_for_block(previous_match_end_in_src - 1, match_start_in_src, src) - after = _get_text_for_block(previous_match_end_in_target - 1, match_start_in_target, target) - return line, before, after - - -def _get_text_for_block(start: int, end: int, lines: List[str]) -> str: - return _replace_invisible_symbols(''.join(lines[start: end])) - - -def _replace_invisible_symbols(line: str) -> str: - for old_str, new_str in zip([" ", "\t", "\n"], ["\u00b7", "\u2192\u2192\u2192\u2192", "\u2193\u000a"]): - line = line.replace(old_str, new_str) - return line - - if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter # see https://github.com/PyCQA/pylint/issues/259 + main() diff --git a/universum/analyzers/utils.py b/universum/analyzers/utils.py index 76e70175..7d921709 100644 --- a/universum/analyzers/utils.py +++ b/universum/analyzers/utils.py @@ -1,11 +1,12 @@ -import json -import sys import argparse import glob +import json +import os import pathlib import subprocess +import sys +from typing import Any, Callable, List, Optional, Tuple, Set, Iterable -from typing import Any, Callable, List, Optional, Tuple, Set from typing_extensions import TypedDict from universum.lib.ci_exception import CiException @@ -19,6 +20,13 @@ def __init__(self, code: int = 2, message: Optional[str] = None): self.message: Optional[str] = message +def create_parser(description: str, module_path: str) -> argparse.ArgumentParser: + module_name, _ = os.path.splitext(os.path.basename(module_path)) + + prog = f"python{sys.version_info.major}.{sys.version_info.minor} -m {__package__}.{module_name}" + return argparse.ArgumentParser(prog=prog, description=description) + + def analyzer(parser: argparse.ArgumentParser): """ Wraps the analyzer specific data and adds common protocol information: @@ -29,6 +37,7 @@ def analyzer(parser: argparse.ArgumentParser): :param parser: Definition of analyzer custom arguments :return: Wrapped analyzer with common reporting behaviour """ + def internal(func: Callable[[argparse.Namespace], List[ReportData]]) -> Callable[[], List[ReportData]]: def wrapper() -> List[ReportData]: add_files_argument(parser) @@ -75,6 +84,7 @@ def sys_exit(func: Callable[[], Any]) -> Callable[[], None]: >>> wrap_system_exit(sys_exit(_raise_custom)) 3 """ + def wrapper() -> None: exit_code: int try: @@ -137,7 +147,7 @@ def add_python_version_argument(parser: argparse.ArgumentParser) -> None: "'python3.7 -m pylint <...>'") -def report_to_file(issues: List[ReportData], json_file: str = None) -> None: +def report_to_file(issues: List[ReportData], json_file: Optional[str] = None) -> None: issues_json = json.dumps(issues, indent=4) if json_file: with open(json_file, "w", encoding="utf-8") as f: @@ -146,6 +156,16 @@ def report_to_file(issues: List[ReportData], json_file: str = None) -> None: sys.stdout.write(issues_json) -def normalize(file: str) -> pathlib.Path: +def normalize_path(file: str) -> pathlib.Path: file_path = pathlib.Path(file) return file_path if file_path.is_absolute() else pathlib.Path.cwd().joinpath(file_path) + + +def get_files_with_absolute_paths(settings: argparse.Namespace) -> Iterable[Tuple[pathlib.Path, + pathlib.Path, + pathlib.Path]]: + for src_file in settings.file_list: + src_file_absolute = normalize_path(src_file) + src_file_relative = src_file_absolute.relative_to(pathlib.Path.cwd()) + target_file_absolute: pathlib.Path = settings.target_folder.joinpath(src_file_relative) + yield src_file_absolute, target_file_absolute, src_file_relative diff --git a/universum/config_creator.py b/universum/config_creator.py index f2f0c53d..a9fbfa70 100644 --- a/universum/config_creator.py +++ b/universum/config_creator.py @@ -1,4 +1,4 @@ -from pathlib import Path +import pathlib import sys from .modules.output.output import MinimalOut @@ -19,7 +19,7 @@ def execute(self) -> None: config_name = ".universum.py" self.out.log(f"Creating an example configuration file '{config_name}'") - config = Path(config_name) + config = pathlib.Path(config_name) config.write_text(f"""#!/usr/bin/env {PYTHON_VERSION} from universum.configuration_support import Configuration, Step diff --git a/universum/configuration_support.py b/universum/configuration_support.py index 3bdb8d11..75fafe40 100644 --- a/universum/configuration_support.py +++ b/universum/configuration_support.py @@ -69,6 +69,12 @@ class Step: execution will not proceed. If no required artifacts were found in the end of the `Universum` run, it is also considered a failure. In case of shell-style patterns build is failed if no files or directories matching pattern are found. + + .. note:: + + Universum checks both the artifact itself and an archive with the same name, because it does not know + in advance whether the step is going to create a file or a directory. + report_artifacts Path to the special artifacts for reporting (e.g. to Swarm). Unlike `artifacts` key, `report_artifacts` are not obligatory and their absence is not considered a build failure. A directory cannot be stored @@ -338,7 +344,7 @@ def get(self, key: str, default: Any = None) -> Any: ... warnings.simplefilter("always") ... f() ... return w - >>> step = Step(name='foo', my_var='bar') + >>> step = Step(name='foo', my_var='bar', t1=None, t2=False) >>> do_and_get_warnings(lambda : step.get('name', 'test')) # doctest: +ELLIPSIS [] @@ -350,12 +356,16 @@ def get(self, key: str, default: Any = None) -> Any: 'test' >>> step.get('command', 'test') 'test' + >>> step.get('t1') is None + True + >>> step.get('t2') + False """ result = self._extras.get(key) - if result: + if result is not None: # for custom fields there is a distinction between None and falsy values return result result = self.__dict__.get(key) - if result: + if result: # non-custom fields initialized with falsy values warn("Using legacy API to access configuration values. Please use var." + key + " instead.") return result return default @@ -672,7 +682,7 @@ def dump(self, produce_string_command: bool = True) -> str: return result def filter(self, checker: Callable[[Step], bool], - parent: Step = None) -> 'Configuration': + parent: Optional[Step] = None) -> 'Configuration': """ This function is supposed to be called from main script, not configuration file. It uses provided `checker` to find all the configurations that pass the check, diff --git a/universum/lib/utils.py b/universum/lib/utils.py index 02154d7f..b924a6e9 100644 --- a/universum/lib/utils.py +++ b/universum/lib/utils.py @@ -104,10 +104,10 @@ def format_traceback(exc: Exception, trace: Optional[TracebackType]) -> str: return tb_text -def catch_exception(exception_name: str, ignore_if: str = None) -> DecoratorT: +def catch_exception(exception_name: str, ignore_if: Optional[str] = None) -> DecoratorT: def decorated_function(function): def function_to_run(*args, **kwargs): - result: ReturnT = None + result: ReturnT = None # type: ignore try: result = function(*args, **kwargs) return result diff --git a/universum/main.py b/universum/main.py index e356da5b..011c934b 100644 --- a/universum/main.py +++ b/universum/main.py @@ -43,6 +43,8 @@ def define_arguments(argument_parser: ModuleArgumentParser) -> None: help="Only applies to build steps where ``code_report=True``; " "disables calculating analysis diff for changed files, " "in this case full analysis report will be published") + argument_parser.add_argument("--fail-unsuccessful", action="store_true", dest="fail_unsuccessful", + help="Return non-zero exit code if any step failed") def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -88,7 +90,9 @@ def execute(self) -> None: self.code_report_collector.repo_diff = repo_diff self.code_report_collector.report_code_report_results() self.artifacts.report_artifacts() - self.reporter.report_build_result() + result = self.reporter.report_build_result() + if self.settings.fail_unsuccessful and not result: + raise SilentAbortException(1) def finalize(self) -> None: if self.settings.no_finalize: diff --git a/universum/modules/artifact_collector.py b/universum/modules/artifact_collector.py index e6a5c094..aaf0bba8 100644 --- a/universum/modules/artifact_collector.py +++ b/universum/modules/artifact_collector.py @@ -1,6 +1,4 @@ import codecs -import distutils -from distutils import dir_util, errors import os import shutil import zipfile @@ -145,12 +143,12 @@ def preprocess_artifact_list(self, artifact_list, ignore_already_existing=False) raise CriticalCiException(text) # Check existence in 'artifacts' directory: wildcards NOT applied - path_to_check1 = os.path.join(self.artifact_dir, os.path.basename(item["path"])) - path_to_check2 = os.path.join(path_to_check1 + ".zip") - if os.path.exists(path_to_check1) or os.path.exists(path_to_check2): - text = f"Build artifact '{os.path.basename(item['path'])}' already present in artifact directory." - text += "\nPossible reason of this error: previous build results in working directory" - raise CriticalCiException(text) + artifact_file = os.path.join(self.artifact_dir, os.path.basename(item["path"])) + self._check_artifact_absent(artifact_file) + + if not self.settings.no_archive: + artifact_zip_archive = os.path.join(artifact_file + ".zip") + self._check_artifact_absent(artifact_zip_archive) @make_block("Preprocessing artifact lists") def set_and_clean_artifacts(self, project_configs: Configuration, ignore_existing_artifacts: bool = False) -> None: @@ -201,11 +199,11 @@ def move_artifact(self, path, is_report=False): # Single file archiving is not implemented at the moment pass try: - distutils.dir_util.copy_tree(matching_path, destination) + shutil.copytree(matching_path, destination) if is_report: text = "'" + artifact_name + "' is not a file and cannot be reported as an artifact" self.out.log(text) - except distutils.errors.DistutilsFileError: + except NotADirectoryError: shutil.copyfile(matching_path, destination) if is_report: artifact_path = self.automation_server.artifact_path(self.artifact_dir, artifact_name) @@ -229,3 +227,10 @@ def clean_artifacts_silently(self): pass os.makedirs(self.artifact_dir) self.html_output.artifact_dir_ready = True + + @staticmethod + def _check_artifact_absent(artifact_path: str): + if os.path.exists(artifact_path): + text: str = f"Build artifact '{os.path.basename(artifact_path)}' already present in artifact directory." + text += "\nPossible reason of this error: previous build results in working directory" + raise CriticalCiException(text) diff --git a/universum/modules/code_report_collector.py b/universum/modules/code_report_collector.py index e558814c..edb2e795 100644 --- a/universum/modules/code_report_collector.py +++ b/universum/modules/code_report_collector.py @@ -92,9 +92,6 @@ def report_code_report_results(self) -> None: if text: report = json.loads(text) - json_file: TextIO = self.artifacts.create_text_file("Static_analysis_report.json") - json_file.write(json.dumps(report, indent=4)) - issue_count: int if not report and report != []: self.out.log_error("There are no results in code report file. Something went wrong.") diff --git a/universum/modules/output/terminal_based_output.py b/universum/modules/output/terminal_based_output.py index f4004d01..980340c4 100644 --- a/universum/modules/output/terminal_based_output.py +++ b/universum/modules/output/terminal_based_output.py @@ -23,7 +23,7 @@ class TerminalBasedOutput(BaseOutput): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.block_level = 0 - self.unicode_acceptable = (locale.getpreferredencoding() == "UTF-8") + self.unicode_acceptable = (locale.getpreferredencoding() == "UTF-8") # pylint: disable = superfluous-parens @staticmethod def _stdout(*args, **kwargs) -> None: diff --git a/universum/modules/reporter.py b/universum/modules/reporter.py index 5c150563..7819bfab 100644 --- a/universum/modules/reporter.py +++ b/universum/modules/reporter.py @@ -1,12 +1,11 @@ from collections import defaultdict from typing import Dict, List, Tuple - from typing_extensions import TypedDict from . import automation_server from .output import HasOutput from .structure_handler import HasStructure, Block -from ..lib.ci_exception import SilentAbortException +from ..lib.ci_exception import CiException from ..lib.gravity import Dependency from ..lib.utils import make_block @@ -54,15 +53,13 @@ def define_arguments(argument_parser): help="Include only the short list of failed steps to reporting comments") parser.add_argument("--report-no-vote", "-rnv", action="store_true", dest="no_vote", help="Do not vote up/down review depending on result") - parser.add_argument("--fail-unsuccessful", "-rfu", action="store_true", dest="fail_unsuccessful", - help="Return non-zero exit code if any step failed") - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self.observers = [] - self.report_initialized = False - self.blocks_to_report = [] - self.artifacts_to_report = [] + self.observers: List = [] + self.report_initialized: bool = False + self.blocks_to_report: List = [] + self.artifacts_to_report: List = [] self.code_report_comments: Dict[str, List[ReportMessage]] = defaultdict(list) self.automation_server = self.automation_server_factory() @@ -101,11 +98,10 @@ def report_artifacts(self, artifact_list): def code_report(self, path: str, message: ReportMessage) -> None: self.code_report_comments[path].append(message) - @make_block("Reporting build result", pass_errors=False) - def report_build_result(self): + def _report_build_result(self) -> bool: if self.report_initialized is False: self.out.log("Not reporting: no build steps executed") - return + return False if self.settings.only_fails_short: self.settings.only_fails = True @@ -122,9 +118,7 @@ def report_build_result(self): if not self.observers: self.out.log("Nowhere to report. Skipping...") - if self.settings.fail_unsuccessful and not is_successful: - raise SilentAbortException(1) - return + return is_successful if is_successful: self.out.log("Reporting successful build...") @@ -132,11 +126,11 @@ def report_build_result(self): text = "Sending comment skipped. " + \ "To report build success, use '--report-build-success' option" self.out.log(text) - text = None + text = "" else: self.out.log("Reporting failed build...") - if text is not None: + if text: if not self.settings.report_start: text += "\n\n" + self.automation_server.report_build_location() @@ -154,8 +148,16 @@ def report_build_result(self): for observer in self.observers: observer.code_report_to_review(self.code_report_comments) - if self.settings.fail_unsuccessful and not is_successful: - raise SilentAbortException(1) + return is_successful + + @make_block("Reporting build result") + def report_build_result(self) -> bool: + try: + return self._report_build_result() + except CiException as e: + self.out.log_error(str(e)) + self.structure.fail_current_block() + return False def _report_steps_recursively(self, block: Block, text: str, indent: str) -> Tuple[str, bool]: has_children: bool = bool(block.children) diff --git a/universum/modules/vcs/perforce_vcs.py b/universum/modules/vcs/perforce_vcs.py index 8974b3d7..e0fc8618 100644 --- a/universum/modules/vcs/perforce_vcs.py +++ b/universum/modules/vcs/perforce_vcs.py @@ -299,7 +299,7 @@ def define_arguments(argument_parser): help="**Revert all vcs within '--p4-client' and delete the workspace.** " "Mandatory for CI environment, otherwise use with caution") - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.check_required_option("client", """ @@ -318,11 +318,11 @@ def __init__(self, *args, **kwargs): self.client_name = self.settings.client self.client_root = self.settings.project_root - self.sync_cls = [] - self.shelve_cls = [] - self.depots = [] - self.client_view = [] - self.mappings_dict = {} + self.sync_cls: List = [] + self.shelve_cls: List = [] + self.depots: List = [] + self.client_view: List = [] + self.mappings_dict: Dict = {} self.unshelved_files: List[Dict[str, str]] = [] self.diff_in_files: List[Tuple[Optional[str], Optional[str], Optional[str]]] = [] diff --git a/universum/modules/vcs/swarm.py b/universum/modules/vcs/swarm.py index 6c7d30ce..ca671301 100644 --- a/universum/modules/vcs/swarm.py +++ b/universum/modules/vcs/swarm.py @@ -9,7 +9,7 @@ from ...lib.ci_exception import CiException from ...lib.gravity import Dependency -urllib3.disable_warnings((urllib3.exceptions.InsecurePlatformWarning, urllib3.exceptions.SNIMissingWarning)) # type: ignore +urllib3.disable_warnings(urllib3.exceptions.InsecurePlatformWarning) # type: ignore __all__ = [ "Swarm" @@ -131,7 +131,7 @@ def update_review_version(self): if self.review_version: return if self.settings.fail_link: - self.review_version = get_version_from_link(self.settings.pass_link) + self.review_version = get_version_from_link(self.settings.fail_link) if self.review_version: return self.out.log("PASS/FAIL links either missing or have unexpected format; " diff --git a/universum/modules/vcs/vcs.py b/universum/modules/vcs/vcs.py index cb31fd08..9884bd6f 100644 --- a/universum/modules/vcs/vcs.py +++ b/universum/modules/vcs/vcs.py @@ -20,7 +20,7 @@ ] -def create_vcs(class_type: str = None) -> Type[ProjectDirectory]: +def create_vcs(class_type: Optional[str] = None) -> Type[ProjectDirectory]: driver_factory_class: Union[ Dict[str, Type[base_vcs.BasePollVcs]], Dict[str, Type[base_vcs.BaseSubmitVcs]],