From ab555ed2fc676537416cd0e2826ad8e0b69ce07b Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Thu, 15 Aug 2019 13:08:57 +0300 Subject: [PATCH 01/32] fix(test): single poll fails because httpretty conflicts with docker-py --- tests/conftest.py | 20 ++++++-------- tests/test_git_poll.py | 61 +++++++++++++++++++++--------------------- tests/test_vcs.py | 49 +++++++++++++++------------------ 3 files changed, 60 insertions(+), 70 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b8a14969..b7712cb5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -60,7 +60,8 @@ def assert_request_was_made(query): return queries.append(request.querystring) - assert False, 'Query string is not found in calls to http server.\nExpected: %s\nActual: %r' % (query, queries) + assert False, 'Query string is not found in calls to http server.\n' \ + 'Expected: %s\nActual: %r' % (query, queries) @staticmethod def assert_request_was_not_made(query): @@ -72,25 +73,20 @@ def assert_request_was_not_made(query): queries.append(request.querystring) @staticmethod - def reset(): - httpretty.disable() + def assert_success_and_collect(function, *args, **kwargs): httpretty.reset() httpretty.enable() httpretty.register_uri(httpretty.GET, "https://localhost/") - @staticmethod - def stop(): - httpretty.disable() - httpretty.reset() + try: + assert function(*args, **kwargs) == 0 + finally: + httpretty.disable() @pytest.fixture() -def http_request_checker(request): - httpretty.enable() - httpretty.register_uri(httpretty.GET, "https://localhost/") +def http_check(request): yield HttpChecker() - httpretty.disable() - httpretty.reset() def check_output((out, err)): diff --git a/tests/test_git_poll.py b/tests/test_git_poll.py index 4c4ad1d8..3a2dda61 100644 --- a/tests/test_git_poll.py +++ b/tests/test_git_poll.py @@ -5,7 +5,6 @@ import pytest import universum -from _universum.poll import Poll from .git_utils import GitEnvironment @@ -32,15 +31,15 @@ def make_branch_with_changes(git_server, branch_name, commits_number, branch_fro return commits -def assert_polled_commits(commits, stdout_checker, http_request_checker): +def assert_polled_commits(commits, stdout_checker, http_check): for commit in commits: stdout_checker.assert_has_calls_with_param("==> Detected commit " + commit) - http_request_checker.assert_request_was_made({"cl": [commit]}) + http_check.assert_request_was_made({"cl": [commit]}) -def test_max_number_commits(stdout_checker, http_request_checker, git_poll_environment): +def test_max_number_commits(stdout_checker, http_check, git_poll_environment): # initialize working directory with initial data - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) # ACT # make changes in polled branch @@ -49,17 +48,17 @@ def test_max_number_commits(stdout_checker, http_request_checker, git_poll_envir # ASSERT # run poll again and trigger the url twice - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) assert_polled_commits(changes_to_polled[1:], stdout_checker, - http_request_checker) + http_check) # Ensure that oldest commit is beyond "allowed_commits_number" and is not polled stdout_checker.assert_absent_calls_with_param("==> Detected commit " + changes_to_polled[0]) -def test_merge_one_branch_ff(stdout_checker, http_request_checker, git_poll_environment): +def test_merge_one_branch_ff(stdout_checker, http_check, git_poll_environment): # initialize working directory with initial data - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) # ACT # make changes in polled branch @@ -71,15 +70,15 @@ def test_merge_one_branch_ff(stdout_checker, http_request_checker, git_poll_envi # ASSERT # run poll again and trigger the url twice - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) assert_polled_commits([change_to_polled] + changes_to_branch, stdout_checker, - http_request_checker) + http_check) -def test_merge_one_branch_noff(stdout_checker, http_request_checker, git_poll_environment): +def test_merge_one_branch_noff(stdout_checker, http_check, git_poll_environment): # initialize working directory with initial data - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) # ACT # make changes in polled branch @@ -92,17 +91,17 @@ def test_merge_one_branch_noff(stdout_checker, http_request_checker, git_poll_en # ASSERT # run poll again and trigger the url twice - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) assert_polled_commits([change_to_polled] + [merge_commit_id], stdout_checker, - http_request_checker) + http_check) stdout_checker.assert_absent_calls_with_param("==> Detected commit " + changes_to_branch[0]) stdout_checker.assert_absent_calls_with_param("==> Detected commit " + changes_to_branch[1]) -def test_merge_two_subsequent_branches_noff(stdout_checker, http_request_checker, git_poll_environment): +def test_merge_two_subsequent_branches_noff(stdout_checker, http_check, git_poll_environment): # initialize working directory with initial data - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) # ACT # make changes in polled branch @@ -117,17 +116,17 @@ def test_merge_two_subsequent_branches_noff(stdout_checker, http_request_checker # ASSERT # run poll again and trigger the url twice - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) assert_polled_commits([change_to_polled] + [merge_commit_id], stdout_checker, - http_request_checker) + http_check) stdout_checker.assert_absent_calls_with_param("==> Detected commit " + changes_to_first_branch[0]) stdout_checker.assert_absent_calls_with_param("==> Detected commit " + changes_to_second_branch[0]) -def test_merge_two_subsequent_branches_ff(stdout_checker, http_request_checker, git_poll_environment): +def test_merge_two_subsequent_branches_ff(stdout_checker, http_check, git_poll_environment): # initialize working directory with initial data - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) # ACT # make changes in polled branch @@ -142,15 +141,15 @@ def test_merge_two_subsequent_branches_ff(stdout_checker, http_request_checker, # ASSERT # run poll again and trigger the url twice - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) assert_polled_commits([change_to_polled] + changes_to_first_branch + changes_to_second_branch, stdout_checker, - http_request_checker) + http_check) -def test_merge_one_branch_noff_1_commit_behind(stdout_checker, http_request_checker, git_poll_environment): +def test_merge_one_branch_noff_1_commit_behind(stdout_checker, http_check, git_poll_environment): # initialize working directory with initial data - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) # ACT # make a branch from polled @@ -168,17 +167,17 @@ def test_merge_one_branch_noff_1_commit_behind(stdout_checker, http_request_chec # ASSERT # run poll again and trigger the url twice - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) assert_polled_commits([change_to_polled, merge_commit_id], stdout_checker, - http_request_checker) + http_check) stdout_checker.assert_absent_calls_with_param("==> Detected commit " + changes_to_branch) @pytest.mark.xfail -def test_merge_ff_commit_merged_from_polled(stdout_checker, http_request_checker, git_poll_environment): +def test_merge_ff_commit_merged_from_polled(stdout_checker, http_check, git_poll_environment): # initialize working directory with initial data - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) # ACT # make a branch from polled @@ -199,8 +198,8 @@ def test_merge_ff_commit_merged_from_polled(stdout_checker, http_request_checker # ASSERT # run poll again and trigger the url twice - assert universum.run(git_poll_environment.settings) == 0 + http_check.assert_success_and_collect(universum.run, git_poll_environment.settings) assert_polled_commits([change_to_polled, change_to_branch], stdout_checker, - http_request_checker) + http_check) stdout_checker.assert_absent_calls_with_param("==> Detected commit " + merge_commit_to_branch) diff --git a/tests/test_vcs.py b/tests/test_vcs.py index a1a5f5df..08b4bc57 100644 --- a/tests/test_vcs.py +++ b/tests/test_vcs.py @@ -39,7 +39,7 @@ def test_git_success_command_line_poll_no_changes(stdout_checker, git_server, tm def test_vcs_error_no_vcs_type(capsys): try: del os.environ["VCS_TYPE"] - except KeyError as e: + except KeyError: pass with pytest.raises(SystemExit): universum.main(["-ot", "term"]) @@ -76,10 +76,10 @@ def test_git_error_command_line_wrong_port(stdout_checker, git_server, tmpdir): class PollerParameters(object): - def __init__(self, log_exception_checker, stdout_checker, http_request_checker, environment): + def __init__(self, log_exception_checker, stdout_checker, http_check, environment): self.log_exception_checker = log_exception_checker self.stdout_checker = stdout_checker - self.http_request_checker = http_request_checker + self.http_check = http_check self.poll_settings = environment.settings self.environment = environment @@ -88,9 +88,9 @@ def make_a_change(self): @pytest.fixture() -def poll_parameters(log_exception_checker, stdout_checker, http_request_checker): +def poll_parameters(log_exception_checker, stdout_checker, http_check): def inner(environment): - return PollerParameters(log_exception_checker, stdout_checker, http_request_checker, environment) + return PollerParameters(log_exception_checker, stdout_checker, http_check, environment) yield inner @@ -106,10 +106,7 @@ def test_poll_error_one_change(poll_parameters, poll_environment): parameters = poll_parameters(poll_environment) # initialize working directory with initial data - assert universum.run(parameters.poll_settings) == 0 - - # stop server - parameters.http_request_checker.stop() + parameters.http_check.assert_success_and_collect(universum.run, parameters.poll_settings) # make change in workspace change = parameters.make_a_change() @@ -127,74 +124,72 @@ def test_poll_success_one_change(poll_parameters, poll_environment): parameters = poll_parameters(poll_environment) # initialize working directory with initial data - assert universum.run(parameters.poll_settings) == 0 + parameters.http_check.assert_success_and_collect(universum.run, parameters.poll_settings) # make change in workspace change = parameters.make_a_change() - # run poll again and trigger the url - assert universum.run(parameters.poll_settings) == 0 + parameters.http_check.assert_success_and_collect(universum.run, parameters.poll_settings) + parameters.http_check.assert_request_was_made({"cl": [change]}) parameters.stdout_checker.assert_has_calls_with_param("==> Detected commit " + change) - parameters.http_request_checker.assert_request_was_made({"cl": [change]}) def test_poll_success_two_changes(poll_parameters, poll_environment): parameters = poll_parameters(poll_environment) # initialize working directory with initial data - assert universum.run(parameters.poll_settings) == 0 + parameters.http_check.assert_success_and_collect(universum.run, parameters.poll_settings) # make changes in workspace change1 = parameters.make_a_change() change2 = parameters.make_a_change() # run poll again and trigger the url twice - assert universum.run(parameters.poll_settings) == 0 + parameters.http_check.assert_success_and_collect(universum.run, parameters.poll_settings) parameters.stdout_checker.assert_has_calls_with_param("==> Detected commit " + change1) parameters.stdout_checker.assert_has_calls_with_param("==> Detected commit " + change2) - parameters.http_request_checker.assert_request_was_made({"cl": [change1]}) - parameters.http_request_checker.assert_request_was_made({"cl": [change2]}) + parameters.http_check.assert_request_was_made({"cl": [change1]}) + parameters.http_check.assert_request_was_made({"cl": [change2]}) def test_poll_changes_several_times(poll_parameters, poll_environment): parameters = poll_parameters(poll_environment) # initialize working directory with initial data - assert universum.run(parameters.poll_settings) == 0 + parameters.http_check.assert_success_and_collect(universum.run, parameters.poll_settings) # make changes in workspace change1 = parameters.make_a_change() change2 = parameters.make_a_change() # run poll and trigger the urls - assert universum.run(parameters.poll_settings) == 0 + parameters.http_check.assert_success_and_collect(universum.run, parameters.poll_settings) parameters.stdout_checker.assert_has_calls_with_param("==> Detected commit " + change1) parameters.stdout_checker.assert_has_calls_with_param("==> Detected commit " + change2) - parameters.http_request_checker.assert_request_was_made({"cl": [change1]}) - parameters.http_request_checker.assert_request_was_made({"cl": [change2]}) + parameters.http_check.assert_request_was_made({"cl": [change1]}) + parameters.http_check.assert_request_was_made({"cl": [change2]}) # make more changes in workspace parameters.stdout_checker.reset() - parameters.http_request_checker.reset() change3 = parameters.make_a_change() change4 = parameters.make_a_change() # run poll and trigger urls for the new changes only - assert universum.run(parameters.poll_settings) == 0 + parameters.http_check.assert_success_and_collect(universum.run, parameters.poll_settings) parameters.stdout_checker.assert_has_calls_with_param("==> Detected commit " + change3) parameters.stdout_checker.assert_has_calls_with_param("==> Detected commit " + change4) parameters.stdout_checker.assert_absent_calls_with_param("==> Detected commit " + change1) parameters.stdout_checker.assert_absent_calls_with_param("==> Detected commit " + change2) - parameters.http_request_checker.assert_request_was_made({"cl": [change3]}) - parameters.http_request_checker.assert_request_was_made({"cl": [change4]}) - parameters.http_request_checker.assert_request_was_not_made({"cl": [change1]}) - parameters.http_request_checker.assert_request_was_not_made({"cl": [change2]}) + parameters.http_check.assert_request_was_made({"cl": [change3]}) + parameters.http_check.assert_request_was_made({"cl": [change4]}) + parameters.http_check.assert_request_was_not_made({"cl": [change1]}) + parameters.http_check.assert_request_was_not_made({"cl": [change2]}) # ---------------------------------------------------------------------------------------------- From 434d9b42e54b9cd0e1d06fdf30be1c24193a9230 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Fri, 16 Aug 2019 18:39:54 +0300 Subject: [PATCH 02/32] fix(test): whitespaces in local path split pip arguments --- tests/deployment_utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/deployment_utils.py b/tests/deployment_utils.py index b8fde893..295fe562 100644 --- a/tests/deployment_utils.py +++ b/tests/deployment_utils.py @@ -97,6 +97,7 @@ def assert_unsuccessful_execution(self, cmd, environment=None): def install_python_module(self, name): if os.path.exists(name): module_name = 'universum' + name = "'" + name + "'" else: module_name = name if not utils.is_pycharm() or self._force_clean: @@ -185,16 +186,16 @@ def set_environment(self, execution_environment): self.environment.install_python_module("coverage") def _basic_args(self): - return " -lo console -pr {} -ad {}".format(self.project_root, self.artifact_dir) + return " -lo console -pr '{}' -ad '{}'".format(self.project_root, self.artifact_dir) def _vcs_args(self, vcs_type): if vcs_type == "none": - return " -vt none --no-diff -fsd {}".format(unicode(self.local.root_directory)) + return " -vt none --no-diff -fsd '{}'".format(unicode(self.local.root_directory)) if vcs_type == "git": - return " -vt git --no-diff -gr {} -grs {}".format(self.git.server.url, self.git.server.target_branch) + return " -vt git --no-diff -gr '{}' -grs '{}'".format(self.git.server.url, self.git.server.target_branch) - return " -vt p4 --p4-force-clean -p4p {} -p4u {} -p4P {} -p4d {} -p4c {}" \ + return " -vt p4 --p4-force-clean -p4p '{}' -p4u '{}' -p4P '{}' -p4d '{}' -p4c {}" \ .format(self.perforce.p4.port, self.perforce.p4.user, self.perforce.p4.password, @@ -209,14 +210,14 @@ def run(self, config, force_installed=False, vcs_type="none", f.write(config) f.close() - cmd = "coverage run --branch --append --source={} {}/universum.py" \ + cmd = "coverage run --branch --append --source='{}' '{}/universum.py'" \ .format(self.working_dir, self.working_dir) # We cannot collect coverage from installed module, so we run it only if specifically told so if force_installed: cmd = "universum" - cmd += " -lcp {}".format(config_file) \ + cmd += " -lcp '{}'".format(config_file) \ + self._basic_args() + self._vcs_args(vcs_type) + additional_parameters if expected_to_fail: @@ -228,7 +229,7 @@ def run(self, config, force_installed=False, vcs_type="none", return result def clean_artifacts(self): - self.environment.assert_successful_execution("rm -rf {}".format(self.artifact_dir)) + self.environment.assert_successful_execution("rm -rf '{}'".format(self.artifact_dir)) @pytest.fixture() From d1dacf2d728102225645431b2231cf2c6b71848c Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Fri, 16 Aug 2019 11:47:05 +0300 Subject: [PATCH 03/32] feat(test): add initial test for GitHub reporter --- tests/conftest.py | 15 +++++++++++++++ tests/test_report.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 tests/test_report.py diff --git a/tests/conftest.py b/tests/conftest.py index b7712cb5..16ce0d6d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,12 @@ # pylint: disable = redefined-outer-name import httpretty +import py import pytest import mock +import utils + pytest_plugins = ['tests.perforce_utils', 'tests.git_utils', 'tests.deployment_utils'] @@ -101,3 +104,15 @@ def detect_fails(capsys): yield capsys check_output(capsys.readouterr()) check_output(capsys._outerr) + + +@pytest.fixture() +def local_sources(tmpdir): + if utils.is_pycharm(): + source_dir = py.path.local(".work").ensure(dir=True) + else: + source_dir = tmpdir.mkdir("project_sources") + local_file = source_dir.join("readme.txt") + local_file.write("This is a an empty file") + + yield utils.Params(root_directory=source_dir, repo_file=local_file) diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 00000000..9975c031 --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# pylint: disable = redefined-outer-name + +import pytest + +import universum, utils + + +class ReportEnvironment(utils.TestEnvironment): + def __init__(self, directory, local_sources): + super(ReportEnvironment, self).__init__(directory, "main") + + self.settings.Vcs.type = "none" + self.settings.MainVcs.report_to_review = True + self.settings.LocalMainVcs.source_dir = unicode(local_sources.root_directory) + + +@pytest.fixture() +def report_environment(tmpdir, local_sources): + yield ReportEnvironment(tmpdir, local_sources) + + +def test_github_run(stdout_checker, http_check, report_environment): + + http_check.assert_success_and_collect(universum.run, report_environment.settings) + + # http_check.assert_request_was_made({"cl": [change]}) + # stdout_checker.assert_has_calls_with_param("==> Detected commit " + change) From 7a9317386899b33afdb66f4d4f9990ccf15ce797 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Fri, 16 Aug 2019 11:48:44 +0300 Subject: [PATCH 04/32] fix(test): remove duplicated fixture --- tests/deployment_utils.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/deployment_utils.py b/tests/deployment_utils.py index 295fe562..8b9c8540 100644 --- a/tests/deployment_utils.py +++ b/tests/deployment_utils.py @@ -4,7 +4,6 @@ import getpass from pwd import getpwnam import os -import py from requests.exceptions import ReadTimeout import docker @@ -146,18 +145,6 @@ def clean_execution_environment(request): runner.exit() -@pytest.fixture() -def local_sources(tmpdir): - if utils.is_pycharm(): - source_dir = py.path.local(".work").ensure(dir=True) - else: - source_dir = tmpdir.mkdir("project_sources") - local_file = source_dir.join("readme.txt") - local_file.write("This is a an empty file") - - yield utils.Params(root_directory=source_dir, repo_file=local_file) - - class UniversumRunner(object): def __init__(self, perforce_workspace, git_client, local_sources): self.perforce = perforce_workspace From 9da42910a14cae505a011bcce3c3d1dbc643f6af Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Fri, 16 Aug 2019 14:21:58 +0300 Subject: [PATCH 05/32] feat(github): add initial 'GitHub' VCS module --- _universum/modules/vcs/github_vcs.py | 56 ++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 _universum/modules/vcs/github_vcs.py diff --git a/_universum/modules/vcs/github_vcs.py b/_universum/modules/vcs/github_vcs.py new file mode 100644 index 00000000..d43cee7e --- /dev/null +++ b/_universum/modules/vcs/github_vcs.py @@ -0,0 +1,56 @@ +# -*- coding: UTF-8 -*- + +from ...lib.gravity import Dependency +from ..reporter import ReportObserver, Reporter +from . import git_vcs + +__all__ = [ + "GithubMainVcs" +] + + +class GithubMainVcs(ReportObserver, git_vcs.GitMainVcs): + """ + This class mostly contains functions for Gihub report observer + """ + reporter_factory = Dependency(Reporter) + + @staticmethod + def define_arguments(argument_parser): + parser = argument_parser.get_or_create_group("Git", "Git repository settings") + + parser.add_argument("--github-token", "-ght", dest="token", metavar="GITHUB_TOKEN", + help="GitHub API token; for details see " + "https://developer.github.com/v3/oauth_authorizations/") + parser.add_argument("--github-check-name", "-ghc", dest="check_name", metavar="GITHUB_CHECK_NAME", + help="The name of Github check run") + + def __init__(self, *args, **kwargs): + super(GithubMainVcs, self).__init__(*args, **kwargs) + self.reporter = None + + def code_review(self): + self.reporter = self.reporter_factory() + self.reporter.subscribe(self) + return self + + def update_review_version(self): + self.out.log("GitHub has no review versions") + + def get_review_link(self): + return self.settings.repo.split(".git") + self.settings.checkout_id + + def is_latest_version(self): + return True + + def code_report_to_review(self, report): + pass + + def report_start(self, report_text): + pass + + def report_result(self, result, report_text=None, no_vote=False): + pass + + def prepare_repository(self): + pass \ No newline at end of file From c22da4a64cce65771fb622b361a34143840c0f73 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Fri, 16 Aug 2019 19:04:20 +0300 Subject: [PATCH 06/32] fix(test): change vcs type --- tests/conftest.py | 15 --------------- tests/deployment_utils.py | 13 +++++++++++++ tests/test_report.py | 11 ++++++----- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 16ce0d6d..b7712cb5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,12 +2,9 @@ # pylint: disable = redefined-outer-name import httpretty -import py import pytest import mock -import utils - pytest_plugins = ['tests.perforce_utils', 'tests.git_utils', 'tests.deployment_utils'] @@ -104,15 +101,3 @@ def detect_fails(capsys): yield capsys check_output(capsys.readouterr()) check_output(capsys._outerr) - - -@pytest.fixture() -def local_sources(tmpdir): - if utils.is_pycharm(): - source_dir = py.path.local(".work").ensure(dir=True) - else: - source_dir = tmpdir.mkdir("project_sources") - local_file = source_dir.join("readme.txt") - local_file.write("This is a an empty file") - - yield utils.Params(root_directory=source_dir, repo_file=local_file) diff --git a/tests/deployment_utils.py b/tests/deployment_utils.py index 8b9c8540..295fe562 100644 --- a/tests/deployment_utils.py +++ b/tests/deployment_utils.py @@ -4,6 +4,7 @@ import getpass from pwd import getpwnam import os +import py from requests.exceptions import ReadTimeout import docker @@ -145,6 +146,18 @@ def clean_execution_environment(request): runner.exit() +@pytest.fixture() +def local_sources(tmpdir): + if utils.is_pycharm(): + source_dir = py.path.local(".work").ensure(dir=True) + else: + source_dir = tmpdir.mkdir("project_sources") + local_file = source_dir.join("readme.txt") + local_file.write("This is a an empty file") + + yield utils.Params(root_directory=source_dir, repo_file=local_file) + + class UniversumRunner(object): def __init__(self, perforce_workspace, git_client, local_sources): self.perforce = perforce_workspace diff --git a/tests/test_report.py b/tests/test_report.py index 9975c031..3f76067a 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -8,17 +8,18 @@ class ReportEnvironment(utils.TestEnvironment): - def __init__(self, directory, local_sources): + def __init__(self, directory, client): super(ReportEnvironment, self).__init__(directory, "main") - self.settings.Vcs.type = "none" + self.settings.Vcs.type = "github" self.settings.MainVcs.report_to_review = True - self.settings.LocalMainVcs.source_dir = unicode(local_sources.root_directory) + self.settings.GitVcs.repo = client.server.url + self.settings.GitVcs.refspec = client.server.target_branch @pytest.fixture() -def report_environment(tmpdir, local_sources): - yield ReportEnvironment(tmpdir, local_sources) +def report_environment(tmpdir, git_client): + yield ReportEnvironment(tmpdir, git_client) def test_github_run(stdout_checker, http_check, report_environment): From 73ba6642c4fe8baf047704a65584b2981a407115 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Tue, 3 Sep 2019 16:26:46 +0300 Subject: [PATCH 07/32] feat(vcs): add 'github' VCS type --- _universum/modules/vcs/github_vcs.py | 19 +++++++++++++------ _universum/modules/vcs/vcs.py | 18 ++++++++++++------ tests/test_report.py | 3 ++- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/_universum/modules/vcs/github_vcs.py b/_universum/modules/vcs/github_vcs.py index d43cee7e..6968ac51 100644 --- a/_universum/modules/vcs/github_vcs.py +++ b/_universum/modules/vcs/github_vcs.py @@ -1,6 +1,7 @@ # -*- coding: UTF-8 -*- from ...lib.gravity import Dependency +from ...lib import utils from ..reporter import ReportObserver, Reporter from . import git_vcs @@ -17,17 +18,26 @@ class GithubMainVcs(ReportObserver, git_vcs.GitMainVcs): @staticmethod def define_arguments(argument_parser): - parser = argument_parser.get_or_create_group("Git", "Git repository settings") + parser = argument_parser.get_or_create_group("GitHub", "GitHub repository settings") parser.add_argument("--github-token", "-ght", dest="token", metavar="GITHUB_TOKEN", help="GitHub API token; for details see " "https://developer.github.com/v3/oauth_authorizations/") parser.add_argument("--github-check-name", "-ghc", dest="check_name", metavar="GITHUB_CHECK_NAME", - help="The name of Github check run") + default="Universum check", help="The name of Github check run") def __init__(self, *args, **kwargs): super(GithubMainVcs, self).__init__(*args, **kwargs) self.reporter = None + utils.check_required_option(self.settings, "checkout_id", """ + git checkout id for github is not specified. + + For github the git checkout id defines the commit to be checked and reported. + Please specify the checkout id by using '--git-checkout-id' ('-gco') + command line parameter or by setting GIT_CHECKOUT_ID environment variable. + + In CI builds commit ID is usually extracted from webhook and handled automatically. + """) def code_review(self): self.reporter = self.reporter_factory() @@ -38,7 +48,7 @@ def update_review_version(self): self.out.log("GitHub has no review versions") def get_review_link(self): - return self.settings.repo.split(".git") + self.settings.checkout_id + return self.settings.repo.split(".git")[0] + self.settings.checkout_id def is_latest_version(self): return True @@ -51,6 +61,3 @@ def report_start(self, report_text): def report_result(self, result, report_text=None, no_vote=False): pass - - def prepare_repository(self): - pass \ No newline at end of file diff --git a/_universum/modules/vcs/vcs.py b/_universum/modules/vcs/vcs.py index 0d7416de..a69ddab7 100644 --- a/_universum/modules/vcs/vcs.py +++ b/_universum/modules/vcs/vcs.py @@ -5,7 +5,7 @@ import shutil import sh -from . import git_vcs, gerrit_vcs, perforce_vcs, local_vcs, base_vcs +from . import git_vcs, github_vcs, gerrit_vcs, perforce_vcs, local_vcs, base_vcs from .. import artifact_collector from ..api_support import ApiSupport from ..project_directory import ProjectDirectory @@ -27,25 +27,29 @@ def create_vcs(class_type=None): p4_driver_factory_class = perforce_vcs.PerforceSubmitVcs git_driver_factory_class = git_vcs.GitSubmitVcs gerrit_driver_factory_class = gerrit_vcs.GerritSubmitVcs + github_driver_factory_class = git_vcs.GitSubmitVcs local_driver_factory_class = base_vcs.BaseSubmitVcs elif class_type == "poll": p4_driver_factory_class = perforce_vcs.PerforcePollVcs git_driver_factory_class = git_vcs.GitPollVcs gerrit_driver_factory_class = git_vcs.GitPollVcs + github_driver_factory_class = git_vcs.GitPollVcs local_driver_factory_class = base_vcs.BasePollVcs else: p4_driver_factory_class = perforce_vcs.PerforceMainVcs git_driver_factory_class = git_vcs.GitMainVcs gerrit_driver_factory_class = gerrit_vcs.GerritMainVcs + github_driver_factory_class = github_vcs.GithubMainVcs local_driver_factory_class = local_vcs.LocalMainVcs - vcs_types = ["none", "p4", "git", "gerrit"] + vcs_types = ["none", "p4", "git", "gerrit", "github"] @needs_structure class Vcs(ProjectDirectory): local_driver_factory = Dependency(local_driver_factory_class) git_driver_factory = Dependency(git_driver_factory_class) gerrit_driver_factory = Dependency(gerrit_driver_factory_class) + github_driver_factory = Dependency(github_driver_factory_class) perforce_driver_factory = Dependency(p4_driver_factory_class) @staticmethod @@ -55,7 +59,7 @@ def define_arguments(argument_parser): parser.add_argument("--vcs-type", "-vt", dest="type", choices=vcs_types, metavar="VCS_TYPE", help="Select repository type to download sources from: Perforce ('p4'), " - "Git ('git'), Gerrit ('gerrit') or a local directory ('none'). " + "Git ('git'), Gerrit ('gerrit'), GitHub ('github') or a local directory ('none'). " "Gerrit uses Git parameters. Each VCS type has its own settings.") def __init__(self, *args, **kwargs): @@ -75,9 +79,9 @@ def __init__(self, *args, **kwargs): Each of these types requires supplying its own configuration parameters. At the minimum, the following parameters are required: - * "git" and "gerrit" - GIT_REPO (-gr) and GIT_REFSPEC (-grs) - * "perforce" - P4PORT (-p4p), P4USER (-p4u) and P4PASSWD (-p4P) - * "none" - SOURCE_DIR (-fsd) + * "git", "github" and "gerrit" - GIT_REPO (-gr) and GIT_REFSPEC (-grs) + * "perforce" - P4PORT (-p4p), P4USER (-p4u) and P4PASSWD (-p4P) + * "none" - SOURCE_DIR (-fsd) Depending on the requested action, additional type-specific parameters are required. For example, P4CLIENT (-p4c) is @@ -91,6 +95,8 @@ def __init__(self, *args, **kwargs): driver_factory = self.git_driver_factory elif self.settings.type == "gerrit": driver_factory = self.gerrit_driver_factory + elif self.settings.type == "github": + driver_factory = self.github_driver_factory else: driver_factory = self.perforce_driver_factory except AttributeError: diff --git a/tests/test_report.py b/tests/test_report.py index 3f76067a..87bc399a 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -14,7 +14,8 @@ def __init__(self, directory, client): self.settings.Vcs.type = "github" self.settings.MainVcs.report_to_review = True self.settings.GitVcs.repo = client.server.url - self.settings.GitVcs.refspec = client.server.target_branch + commit_id = unicode(client.repo.remotes.origin.refs[client.server.target_branch].commit) + self.settings.GitMainVcs.checkout_id = commit_id @pytest.fixture() From 39ac4f2b237f40f1eb4e357d1b920a7ea4fd8e8d Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Tue, 3 Sep 2019 16:34:43 +0300 Subject: [PATCH 08/32] feat(test): add argument checks for 'github' vcs type --- tests/test_argument_check.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_argument_check.py b/tests/test_argument_check.py index c62b42a5..c910cb2a 100644 --- a/tests/test_argument_check.py +++ b/tests/test_argument_check.py @@ -29,13 +29,16 @@ def create_settings(test_type, vcs_type): settings.TeamcityServer.user_id = "TeamCityUser" settings.TeamcityServer.passwd = "TeamCityPassword" - if vcs_type == "git" or vcs_type == "gerrit": + if vcs_type == "git" or vcs_type == "gerrit" or vcs_type == "github": settings.GitVcs.repo = "ssh://user@127.0.0.1" - # the refspec is crafted to satisfy requirements of our gerrit module, and git doesn't care + # the refspec is crafted to satisfy requirements of our gerrit module, and others don't care settings.GitVcs.refspec = "refs/changes/00/000000/0" if test_type == "submit": settings.GitSubmitVcs.user = "Testing User" settings.GitSubmitVcs.email = "some@email.com" + elif test_type == "main": + # the checkout id is crafted to satisfy requirements of our github module, and others don't care + settings.GitMainVcs.checkout_id = "HEAD" elif vcs_type == "none": if test_type == "main": settings.LocalMainVcs.source_dir = "temp" @@ -89,7 +92,7 @@ def param(test_type, module, field, vcs_type="*", error_match=None): return if vcs_type == "*": - param(test_type, module, field, ["p4", "git", "gerrit", "none"], error_match) + param(test_type, module, field, ["p4", "git", "gerrit", "github", "none"], error_match) return if test_type == "*": @@ -126,6 +129,7 @@ def param(test_type, module, field, vcs_type="*", error_match=None): param("*", "Vcs", "type") param("*", "GitVcs", "repo", vcs_type=["git", "gerrit"]) param("main", "GitVcs", "refspec", vcs_type="gerrit") +param("main", "GitMainVcs", "checkout_id", vcs_type="github") # pylint: enable = bad-whitespace From 8f63d4bf3a3bd20ad4e32676147cbaf99101c22e Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Tue, 3 Sep 2019 17:59:10 +0300 Subject: [PATCH 09/32] fix(test): git checkout id cannot be set for gerrit vcs_type --- tests/test_argument_check.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_argument_check.py b/tests/test_argument_check.py index c910cb2a..6198fcc1 100644 --- a/tests/test_argument_check.py +++ b/tests/test_argument_check.py @@ -36,8 +36,7 @@ def create_settings(test_type, vcs_type): if test_type == "submit": settings.GitSubmitVcs.user = "Testing User" settings.GitSubmitVcs.email = "some@email.com" - elif test_type == "main": - # the checkout id is crafted to satisfy requirements of our github module, and others don't care + elif test_type == "main" and vcs_type != "gerrit": settings.GitMainVcs.checkout_id = "HEAD" elif vcs_type == "none": if test_type == "main": From 2c7cd0a48501b257b961c587f438531686157993 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Thu, 5 Sep 2019 13:22:42 +0300 Subject: [PATCH 10/32] fix(test): add missing argument checks for github --- tests/test_argument_check.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_argument_check.py b/tests/test_argument_check.py index 6198fcc1..e1589859 100644 --- a/tests/test_argument_check.py +++ b/tests/test_argument_check.py @@ -108,12 +108,12 @@ def param(test_type, module, field, vcs_type="*", error_match=None): # pylint: disable = bad-whitespace param("main", "Launcher", "config_path") -param("submit", "Submit", "commit_message", vcs_type=["p4", "git", "gerrit"]) +param("submit", "Submit", "commit_message", vcs_type=["p4", "git", "gerrit", "github"]) param("submit", "PerforceSubmitVcs", "client", vcs_type="p4") param("main", "PerforceMainVcs", "client", vcs_type="p4") param("main", "LocalMainVcs", "source_dir", vcs_type="none") -param("submit", "GitSubmitVcs", "user", vcs_type=["git", "gerrit"]) -param("submit", "GitSubmitVcs", "email", vcs_type=["git", "gerrit"]) +param("submit", "GitSubmitVcs", "user", vcs_type=["git", "gerrit", "github"]) +param("submit", "GitSubmitVcs", "email", vcs_type=["git", "gerrit", "github"]) param("main", "TeamcityServer", "build_id") param("main", "TeamcityServer", "configuration_id") param("main", "TeamcityServer", "server_url", error_match="TEAMCITY_SERVER") @@ -126,7 +126,7 @@ def param(test_type, module, field, vcs_type="*", error_match=None): param("main", "Swarm", "review_id", vcs_type="p4", error_match="REVIEW") param("main", "Swarm", "change", vcs_type="p4") param("*", "Vcs", "type") -param("*", "GitVcs", "repo", vcs_type=["git", "gerrit"]) +param("*", "GitVcs", "repo", vcs_type=["git", "gerrit", "github"]) param("main", "GitVcs", "refspec", vcs_type="gerrit") param("main", "GitMainVcs", "checkout_id", vcs_type="github") # pylint: enable = bad-whitespace From 9a42932084aea4fad9166373f8f9b7f6525b4873 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Wed, 11 Sep 2019 17:28:04 +0300 Subject: [PATCH 11/32] feat(github): add initial reporting build start and finish --- _universum/lib/utils.py | 11 +++++- _universum/modules/vcs/github_vcs.py | 56 ++++++++++++++++++++++++++-- _universum/modules/vcs/swarm.py | 11 +----- tests/conftest.py | 10 +++-- tests/test_report.py | 24 +++++++++--- 5 files changed, 90 insertions(+), 22 deletions(-) diff --git a/_universum/lib/utils.py b/_universum/lib/utils.py index d169caa8..8d43d988 100644 --- a/_universum/lib/utils.py +++ b/_universum/lib/utils.py @@ -7,6 +7,7 @@ import sys import traceback +from _universum.lib.ci_exception import CiException from .ci_exception import CriticalCiException, SilentAbortException from .module_arguments import IncorrectParameterError @@ -24,7 +25,8 @@ "convert_to_str", "unify_argument_list", "Uninterruptible", - "make_block" + "make_block", + "check_request_result" ] # For proper unicode symbols processing @@ -208,3 +210,10 @@ def function_in_block(self, *args, **kwargs): return self.structure.run_in_block(func, block_name, pass_errors, self, *args, **kwargs) return function_in_block return decorated_function + + +def check_request_result(result): + if result.status_code != 200: + text = "Invalid return code " + result.status_code + ". Response is:\n" + text += result.text + raise CiException(text) diff --git a/_universum/modules/vcs/github_vcs.py b/_universum/modules/vcs/github_vcs.py index 6968ac51..7c3038b1 100644 --- a/_universum/modules/vcs/github_vcs.py +++ b/_universum/modules/vcs/github_vcs.py @@ -1,5 +1,9 @@ # -*- coding: UTF-8 -*- +import datetime +import requests +import urlparse + from ...lib.gravity import Dependency from ...lib import utils from ..reporter import ReportObserver, Reporter @@ -25,6 +29,8 @@ def define_arguments(argument_parser): "https://developer.github.com/v3/oauth_authorizations/") parser.add_argument("--github-check-name", "-ghc", dest="check_name", metavar="GITHUB_CHECK_NAME", default="Universum check", help="The name of Github check run") + parser.add_argument("--github-api-url", "-gha", dest="api_url", metavar="GITHUB_API_URL", + default="https://api.github.com/", help="API URL for integration") def __init__(self, *args, **kwargs): super(GithubMainVcs, self).__init__(*args, **kwargs) @@ -39,6 +45,9 @@ def __init__(self, *args, **kwargs): In CI builds commit ID is usually extracted from webhook and handled automatically. """) + parsed_repo = urlparse.urlparse(self.settings.repo) + self.repo_path = unicode(parsed_repo.path).strip(".git") + def code_review(self): self.reporter = self.reporter_factory() self.reporter.subscribe(self) @@ -48,7 +57,7 @@ def update_review_version(self): self.out.log("GitHub has no review versions") def get_review_link(self): - return self.settings.repo.split(".git")[0] + self.settings.checkout_id + return self.settings.repo.split(".git")[0] + "/" + self.settings.checkout_id def is_latest_version(self): return True @@ -57,7 +66,48 @@ def code_report_to_review(self, report): pass def report_start(self, report_text): - pass + headers = { + "Accept": "application/vnd.github.antiope-preview+json", + "Authorization": "token " + self.settings.token + } + request = { + "name": self.settings.check_name, + "status": "in_progress", + "started_at": datetime.datetime.now().isoformat(), + "output": { + "title": self.settings.check_name, + "summary": report_text + } + } + + path = self.settings.api_url + "repos" + self.repo_path + "/check-runs" + result = requests.post(path, data=request, headers=headers) + utils.check_request_result(result) def report_result(self, result, report_text=None, no_vote=False): - pass + headers = { + "Accept": "application/vnd.github.antiope-preview+json", + "Authorization": "token " + self.settings.token + } + if result: + conclusion = "success" + else: + conclusion = "failure" + + if not report_text: + report_text = "" + + request = { + "name": self.settings.check_name, + "status": "completed", + "completed_at": datetime.datetime.now().isoformat(), + "conclusion": conclusion, + "output": { + "title": self.settings.check_name, + "summary": report_text + } + } + + path = self.settings.api_url + "repos" + self.repo_path + "/check-runs" + result = requests.post(path, data=request, headers=headers) + utils.check_request_result(result) diff --git a/_universum/modules/vcs/swarm.py b/_universum/modules/vcs/swarm.py index ad0c7930..228f9c15 100644 --- a/_universum/modules/vcs/swarm.py +++ b/_universum/modules/vcs/swarm.py @@ -20,13 +20,6 @@ ] -def check_request_result(result): - if result.status_code != 200: - text = "Invalid return code " + result.status_code + ". Response is:\n" - text += result.text - raise CiException(text) - - @needs_output class Swarm(ReportObserver, Module): """ @@ -154,7 +147,7 @@ def post_comment(self, text, filename=None, line=None, version=None, no_notifica result = requests.post(self.settings.server_url + "/api/v9/comments", data=request, auth=(self.user, self.password)) - check_request_result(result) + utils.check_request_result(result) def vote_review(self, result, version=None): request = {} @@ -167,7 +160,7 @@ def vote_review(self, result, version=None): result = requests.patch(self.settings.server_url + "/api/v6/reviews/" + self.settings.review_id, data=request, auth=(self.user, self.password)) - check_request_result(result) + utils.check_request_result(result) def report_start(self, report_text): self.update_review_version() diff --git a/tests/conftest.py b/tests/conftest.py index b7712cb5..4388e38e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -73,13 +73,17 @@ def assert_request_was_not_made(query): queries.append(request.querystring) @staticmethod - def assert_success_and_collect(function, *args, **kwargs): + def assert_success_and_collect(function, params, url="https://localhost/", method="GET"): httpretty.reset() httpretty.enable() - httpretty.register_uri(httpretty.GET, "https://localhost/") + if method == "GET": + hmethod = httpretty.GET + else: + hmethod = httpretty.POST + httpretty.register_uri(hmethod, url) try: - assert function(*args, **kwargs) == 0 + assert function(params) == 0 finally: httpretty.disable() diff --git a/tests/test_report.py b/tests/test_report.py index 87bc399a..7086288b 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -2,9 +2,11 @@ # -*- coding: UTF-8 -*- # pylint: disable = redefined-outer-name +import httpretty import pytest -import universum, utils +import universum +import utils class ReportEnvironment(utils.TestEnvironment): @@ -16,6 +18,14 @@ def __init__(self, directory, client): self.settings.GitVcs.repo = client.server.url commit_id = unicode(client.repo.remotes.origin.refs[client.server.target_branch].commit) self.settings.GitMainVcs.checkout_id = commit_id + self.settings.GithubMainVcs.token = "token" + self.settings.GithubMainVcs.api_url = "http://api.github.com/" + self.settings.Reporter.report_start = True + self.settings.Reporter.report_success = True + + self.path = "http://api.github.com/repos" + \ + unicode(client.root_directory).strip("client") + \ + "server/check-runs" @pytest.fixture() @@ -23,9 +33,11 @@ def report_environment(tmpdir, git_client): yield ReportEnvironment(tmpdir, git_client) -def test_github_run(stdout_checker, http_check, report_environment): - - http_check.assert_success_and_collect(universum.run, report_environment.settings) - - # http_check.assert_request_was_made({"cl": [change]}) +# def test_github_run(stdout_checker, http_check, report_environment): +def test_github_run(http_check, report_environment): + http_check.assert_success_and_collect(universum.run, report_environment.settings, + url=report_environment.path, method="POST") + requests = httpretty.httpretty.latest_requests + print requests + # http_check.assert_request_was_made({"status": "completed"}) # stdout_checker.assert_has_calls_with_param("==> Detected commit " + change) From 10cf841ed15aff26df6426e8aa531bf1be2c9cb5 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Thu, 12 Sep 2019 14:51:22 +0300 Subject: [PATCH 12/32] update(docs): update version to differ from latest release --- _universum/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_universum/__init__.py b/_universum/__init__.py index 6eb2d765..0d735057 100644 --- a/_universum/__init__.py +++ b/_universum/__init__.py @@ -1,4 +1,4 @@ # -*- coding: UTF-8 -*- __title__ = "Universum" -__version__ = "0.18.1" +__version__ = "0.18.2" From 3a2bb6b5d909b4a57561541ea0e3829e4a26f8b5 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Thu, 12 Sep 2019 15:33:47 +0300 Subject: [PATCH 13/32] fix(github): make 'token' parameter required --- _universum/modules/vcs/github_vcs.py | 12 ++++++++++++ tests/test_argument_check.py | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/_universum/modules/vcs/github_vcs.py b/_universum/modules/vcs/github_vcs.py index 7c3038b1..720d0f05 100644 --- a/_universum/modules/vcs/github_vcs.py +++ b/_universum/modules/vcs/github_vcs.py @@ -44,6 +44,18 @@ def __init__(self, *args, **kwargs): In CI builds commit ID is usually extracted from webhook and handled automatically. """) + utils.check_required_option(self.settings, "token", """ + github api token is not specified. + + GitHub API token is used for application authorization. Expected format is + 'v1.4e96f8c2d1922c3b154a65ca7ecb91f6994fb0c5'. + Please specify the token by using '--github-token' ('-ght') + command line parameter or by setting GITHUB_TOKEN environment variable. + + In CI builds github api token is expected to be acquired automatically before build. + For information on how to acquire the token please see + https://developer.github.com/v3/oauth_authorizations/ + """) parsed_repo = urlparse.urlparse(self.settings.repo) self.repo_path = unicode(parsed_repo.path).strip(".git") diff --git a/tests/test_argument_check.py b/tests/test_argument_check.py index e1589859..6e8691e3 100644 --- a/tests/test_argument_check.py +++ b/tests/test_argument_check.py @@ -36,8 +36,9 @@ def create_settings(test_type, vcs_type): if test_type == "submit": settings.GitSubmitVcs.user = "Testing User" settings.GitSubmitVcs.email = "some@email.com" - elif test_type == "main" and vcs_type != "gerrit": + elif test_type == "main" and vcs_type == "github": settings.GitMainVcs.checkout_id = "HEAD" + settings.GithubMainVcs.token = "some_token" elif vcs_type == "none": if test_type == "main": settings.LocalMainVcs.source_dir = "temp" @@ -129,6 +130,7 @@ def param(test_type, module, field, vcs_type="*", error_match=None): param("*", "GitVcs", "repo", vcs_type=["git", "gerrit", "github"]) param("main", "GitVcs", "refspec", vcs_type="gerrit") param("main", "GitMainVcs", "checkout_id", vcs_type="github") +param("main", "GithubMainVcs", "token", vcs_type="github") # pylint: enable = bad-whitespace From 6272b19027ef369fcb3e57d6db87d910df06eb49 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Thu, 12 Sep 2019 17:15:29 +0300 Subject: [PATCH 14/32] fix(github): add missing parameter for check run id --- _universum/modules/vcs/github_vcs.py | 24 +++++++++++++++--------- tests/test_argument_check.py | 2 ++ tests/test_report.py | 3 ++- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/_universum/modules/vcs/github_vcs.py b/_universum/modules/vcs/github_vcs.py index 720d0f05..b1fc5a2d 100644 --- a/_universum/modules/vcs/github_vcs.py +++ b/_universum/modules/vcs/github_vcs.py @@ -29,6 +29,8 @@ def define_arguments(argument_parser): "https://developer.github.com/v3/oauth_authorizations/") parser.add_argument("--github-check-name", "-ghc", dest="check_name", metavar="GITHUB_CHECK_NAME", default="Universum check", help="The name of Github check run") + parser.add_argument("--github-check-id", "-ghi", dest="check_id", metavar="GITHUB_CHECK_ID", + help="Github check run ID") parser.add_argument("--github-api-url", "-gha", dest="api_url", metavar="GITHUB_API_URL", default="https://api.github.com/", help="API URL for integration") @@ -56,9 +58,17 @@ def __init__(self, *args, **kwargs): For information on how to acquire the token please see https://developer.github.com/v3/oauth_authorizations/ """) + utils.check_required_option(self.settings, "check_id", """ + github check id is not specified. - parsed_repo = urlparse.urlparse(self.settings.repo) - self.repo_path = unicode(parsed_repo.path).strip(".git") + GitHub check runs each have unique ID, that is used to update check result. + To integrate Universum with GitHub, check run should be already created before performing + actual check. Please specify check run ID by using '--github-check-id' ('-ghi') + command line parameter or by setting GITHUB_CHECK_ID environment variable. + """) + + repo_path = unicode(urlparse.urlparse(self.settings.repo).path).rsplit(".git", 1)[0] + self.check_url = self.settings.api_url + "repos" + repo_path + "/check-runs/" + self.settings.check_id def code_review(self): self.reporter = self.reporter_factory() @@ -69,7 +79,7 @@ def update_review_version(self): self.out.log("GitHub has no review versions") def get_review_link(self): - return self.settings.repo.split(".git")[0] + "/" + self.settings.checkout_id + return self.settings.repo.rsplit(".git", 1)[0] + "/" + "pulls" def is_latest_version(self): return True @@ -83,7 +93,6 @@ def report_start(self, report_text): "Authorization": "token " + self.settings.token } request = { - "name": self.settings.check_name, "status": "in_progress", "started_at": datetime.datetime.now().isoformat(), "output": { @@ -92,8 +101,7 @@ def report_start(self, report_text): } } - path = self.settings.api_url + "repos" + self.repo_path + "/check-runs" - result = requests.post(path, data=request, headers=headers) + result = requests.post(self.check_url, data=request, headers=headers) utils.check_request_result(result) def report_result(self, result, report_text=None, no_vote=False): @@ -110,7 +118,6 @@ def report_result(self, result, report_text=None, no_vote=False): report_text = "" request = { - "name": self.settings.check_name, "status": "completed", "completed_at": datetime.datetime.now().isoformat(), "conclusion": conclusion, @@ -120,6 +127,5 @@ def report_result(self, result, report_text=None, no_vote=False): } } - path = self.settings.api_url + "repos" + self.repo_path + "/check-runs" - result = requests.post(path, data=request, headers=headers) + result = requests.post(self.check_url, data=request, headers=headers) utils.check_request_result(result) diff --git a/tests/test_argument_check.py b/tests/test_argument_check.py index 6e8691e3..c86518b7 100644 --- a/tests/test_argument_check.py +++ b/tests/test_argument_check.py @@ -39,6 +39,7 @@ def create_settings(test_type, vcs_type): elif test_type == "main" and vcs_type == "github": settings.GitMainVcs.checkout_id = "HEAD" settings.GithubMainVcs.token = "some_token" + settings.GithubMainVcs.check_id = "000000000" elif vcs_type == "none": if test_type == "main": settings.LocalMainVcs.source_dir = "temp" @@ -131,6 +132,7 @@ def param(test_type, module, field, vcs_type="*", error_match=None): param("main", "GitVcs", "refspec", vcs_type="gerrit") param("main", "GitMainVcs", "checkout_id", vcs_type="github") param("main", "GithubMainVcs", "token", vcs_type="github") +param("main", "GithubMainVcs", "check_id", vcs_type="github") # pylint: enable = bad-whitespace diff --git a/tests/test_report.py b/tests/test_report.py index 7086288b..c6fac18b 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -19,12 +19,13 @@ def __init__(self, directory, client): commit_id = unicode(client.repo.remotes.origin.refs[client.server.target_branch].commit) self.settings.GitMainVcs.checkout_id = commit_id self.settings.GithubMainVcs.token = "token" + self.settings.GithubMainVcs.check_id = "123" self.settings.GithubMainVcs.api_url = "http://api.github.com/" self.settings.Reporter.report_start = True self.settings.Reporter.report_success = True self.path = "http://api.github.com/repos" + \ - unicode(client.root_directory).strip("client") + \ + unicode(client.root_directory).rsplit("client", 0)[0] + \ "server/check-runs" From 3ab9209ec7a1cde594a9a661facf0cdf42f00719 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Thu, 12 Sep 2019 17:29:15 +0300 Subject: [PATCH 15/32] fix(utils): moved function had integer status code instead of unicode --- _universum/lib/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_universum/lib/utils.py b/_universum/lib/utils.py index 8d43d988..b22020c5 100644 --- a/_universum/lib/utils.py +++ b/_universum/lib/utils.py @@ -214,6 +214,6 @@ def function_in_block(self, *args, **kwargs): def check_request_result(result): if result.status_code != 200: - text = "Invalid return code " + result.status_code + ". Response is:\n" + text = "Invalid return code " + unicode(result.status_code) + ". Response is:\n" text += result.text raise CiException(text) From 0624698dad79e36a0770eedae67ab45abb8719e5 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Thu, 12 Sep 2019 18:07:33 +0300 Subject: [PATCH 16/32] fix(github): fix time format, change POST requests to PATCH --- _universum/modules/vcs/github_vcs.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/_universum/modules/vcs/github_vcs.py b/_universum/modules/vcs/github_vcs.py index b1fc5a2d..fda5e5fd 100644 --- a/_universum/modules/vcs/github_vcs.py +++ b/_universum/modules/vcs/github_vcs.py @@ -14,6 +14,10 @@ ] +def get_time(): + return datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z" + + class GithubMainVcs(ReportObserver, git_vcs.GitMainVcs): """ This class mostly contains functions for Gihub report observer @@ -94,7 +98,7 @@ def report_start(self, report_text): } request = { "status": "in_progress", - "started_at": datetime.datetime.now().isoformat(), + "started_at": get_time(), "output": { "title": self.settings.check_name, "summary": report_text @@ -119,7 +123,7 @@ def report_result(self, result, report_text=None, no_vote=False): request = { "status": "completed", - "completed_at": datetime.datetime.now().isoformat(), + "completed_at": get_time(), "conclusion": conclusion, "output": { "title": self.settings.check_name, From bc874e0c6bc38783092b9fe58deae0347716c78d Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Thu, 12 Sep 2019 18:14:59 +0300 Subject: [PATCH 17/32] fix(github): change POST requests to PATCH, change 'data' to 'json' --- _universum/modules/vcs/github_vcs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_universum/modules/vcs/github_vcs.py b/_universum/modules/vcs/github_vcs.py index fda5e5fd..e9297d8e 100644 --- a/_universum/modules/vcs/github_vcs.py +++ b/_universum/modules/vcs/github_vcs.py @@ -105,7 +105,7 @@ def report_start(self, report_text): } } - result = requests.post(self.check_url, data=request, headers=headers) + result = requests.patch(self.check_url, json=request, headers=headers) utils.check_request_result(result) def report_result(self, result, report_text=None, no_vote=False): @@ -131,5 +131,5 @@ def report_result(self, result, report_text=None, no_vote=False): } } - result = requests.post(self.check_url, data=request, headers=headers) + result = requests.patch(self.check_url, json=request, headers=headers) utils.check_request_result(result) From 2a8ba7c85f7f6ae81721d8c77701e8b3b0bb1e4d Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Fri, 13 Sep 2019 11:56:49 +0300 Subject: [PATCH 18/32] fix(test): add PATCH method to http_check --- tests/conftest.py | 4 +++- tests/test_report.py | 14 ++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4388e38e..dbf25ece 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -78,8 +78,10 @@ def assert_success_and_collect(function, params, url="https://localhost/", metho httpretty.enable() if method == "GET": hmethod = httpretty.GET - else: + elif method == "POST": hmethod = httpretty.POST + else: + hmethod = httpretty.PATCH httpretty.register_uri(hmethod, url) try: diff --git a/tests/test_report.py b/tests/test_report.py index c6fac18b..1140843a 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -20,13 +20,13 @@ def __init__(self, directory, client): self.settings.GitMainVcs.checkout_id = commit_id self.settings.GithubMainVcs.token = "token" self.settings.GithubMainVcs.check_id = "123" - self.settings.GithubMainVcs.api_url = "http://api.github.com/" + self.settings.GithubMainVcs.api_url = "http://localhost/" self.settings.Reporter.report_start = True self.settings.Reporter.report_success = True - self.path = "http://api.github.com/repos" + \ - unicode(client.root_directory).rsplit("client", 0)[0] + \ - "server/check-runs" + self.path = "http://localhost/repos" + \ + unicode(client.root_directory).rsplit("client", 1)[0] + \ + "server/check-runs/123" @pytest.fixture() @@ -36,9 +36,11 @@ def report_environment(tmpdir, git_client): # def test_github_run(stdout_checker, http_check, report_environment): def test_github_run(http_check, report_environment): + print report_environment.path http_check.assert_success_and_collect(universum.run, report_environment.settings, - url=report_environment.path, method="POST") + url=report_environment.path, method="PATCH") requests = httpretty.httpretty.latest_requests - print requests + for req in requests: + print req # http_check.assert_request_was_made({"status": "completed"}) # stdout_checker.assert_has_calls_with_param("==> Detected commit " + change) From 1c38aefd3d7a1802f40aa22f6a45de2e28356b3d Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Fri, 13 Sep 2019 14:41:08 +0300 Subject: [PATCH 19/32] fix(github): change review link to exact check run --- _universum/modules/vcs/github_vcs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_universum/modules/vcs/github_vcs.py b/_universum/modules/vcs/github_vcs.py index e9297d8e..96c24dc8 100644 --- a/_universum/modules/vcs/github_vcs.py +++ b/_universum/modules/vcs/github_vcs.py @@ -83,7 +83,7 @@ def update_review_version(self): self.out.log("GitHub has no review versions") def get_review_link(self): - return self.settings.repo.rsplit(".git", 1)[0] + "/" + "pulls" + return self.settings.repo.rsplit(".git", 1)[0] + "/runs/" + self.settings.check_id def is_latest_version(self): return True From 1ae2dc3e373cdfe637fa034d00e210f8bff9cf9b Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Fri, 13 Sep 2019 16:04:54 +0300 Subject: [PATCH 20/32] feat(test): add POST body checks to http_check --- tests/conftest.py | 16 ++++++++++++++++ tests/test_report.py | 9 ++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index dbf25ece..1393b364 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -72,6 +72,22 @@ def assert_request_was_not_made(query): '\nExpected: %s\nActual: %r' % (query, queries) queries.append(request.querystring) + @staticmethod + def assert_request_body_contained(key, value): + results = [] + for request in httpretty.httpretty.latest_requests: + if key in request.parsed_body: + if request.parsed_body[key] == value: + return + results.append(request.parsed_body[key]) + + if not results: + text = "No requests with field '{}' found in calls to http server".format(key) + else: + text = "No requests with field '{}' set to '{}' found in calls to http server.\n" \ + "However, requests with following values were made: {}".format(key, value, results) + assert False, text + @staticmethod def assert_success_and_collect(function, params, url="https://localhost/", method="GET"): httpretty.reset() diff --git a/tests/test_report.py b/tests/test_report.py index 1140843a..de0a0867 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -36,11 +36,10 @@ def report_environment(tmpdir, git_client): # def test_github_run(stdout_checker, http_check, report_environment): def test_github_run(http_check, report_environment): - print report_environment.path http_check.assert_success_and_collect(universum.run, report_environment.settings, url=report_environment.path, method="PATCH") - requests = httpretty.httpretty.latest_requests - for req in requests: - print req - # http_check.assert_request_was_made({"status": "completed"}) + + http_check.assert_request_body_contained("status", "in_progress") + http_check.assert_request_body_contained("status", "completed") + http_check.assert_request_body_contained("conclusion", "success") # stdout_checker.assert_has_calls_with_param("==> Detected commit " + change) From 06da4adfcdf336f9e177186f3fdbc5b87fcc9975 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Tue, 17 Sep 2019 17:56:14 +0300 Subject: [PATCH 21/32] fix(config): add quotes for whitespaces in path --- tests/configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/configs.py b/tests/configs.py index 6e5ac83e..71bb6461 100644 --- a/tests/configs.py +++ b/tests/configs.py @@ -8,7 +8,7 @@ def run_virtual(cmd): pylint_cmd = "universum_pylint --python-version 2 --rcfile pylintrc " + \ - "--files *.py _universum/ tests/ analyzers/ --result-file ${CODE_REPORT_FILE}" + "--files *.py _universum/ tests/ analyzers/ --result-file '${CODE_REPORT_FILE}'" configs = Variations([dict(name="Update Docker images", command=["make", "images"]), dict(name="Create virtual environment", From a94cef4ac0215fab713cf27ccca5bd1a3af7134f Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Tue, 17 Sep 2019 19:46:05 +0300 Subject: [PATCH 22/32] fix(github): add authentication via token for HTTPS --- _universum/modules/vcs/git_vcs.py | 9 +++++---- _universum/modules/vcs/github_vcs.py | 7 ++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/_universum/modules/vcs/git_vcs.py b/_universum/modules/vcs/git_vcs.py index 62db7d63..ed7e603d 100644 --- a/_universum/modules/vcs/git_vcs.py +++ b/_universum/modules/vcs/git_vcs.py @@ -70,6 +70,7 @@ def __init__(self, out, *args, **kwargs): def line_dropped(self, line): self.out.log(line) + self.clone_url = self.settings.repo self.repo = None self.logger = Progress(self.out) @@ -84,16 +85,16 @@ def line_dropped(self, line): @make_block("Cloning repository") @catch_git_exception() def clone_and_fetch(self, history_depth=None): - self.out.log("Cloning '" + self.settings.repo + "'...") + self.out.log("Cloning '" + self.clone_url + "'...") destination_directory = convert_to_str(self.settings.project_root) if history_depth: - self.repo = git.Repo.clone_from(self.settings.repo, destination_directory, + self.repo = git.Repo.clone_from(self.clone_url, destination_directory, depth=history_depth, no_single_branch=True, progress=self.logger) else: - self.repo = git.Repo.clone_from(self.settings.repo, destination_directory, progress=self.logger) + self.repo = git.Repo.clone_from(self.clone_url, destination_directory, progress=self.logger) self.sources_need_cleaning = True - self.append_repo_status("Git repo: " + self.settings.repo + "\n\n") + self.append_repo_status("Git repo: " + self.clone_url + "\n\n") self.out.log("Please note that default remote name is 'origin'") if self.settings.refspec: diff --git a/_universum/modules/vcs/github_vcs.py b/_universum/modules/vcs/github_vcs.py index 96c24dc8..d8a80554 100644 --- a/_universum/modules/vcs/github_vcs.py +++ b/_universum/modules/vcs/github_vcs.py @@ -71,8 +71,13 @@ def __init__(self, *args, **kwargs): command line parameter or by setting GITHUB_CHECK_ID environment variable. """) - repo_path = unicode(urlparse.urlparse(self.settings.repo).path).rsplit(".git", 1)[0] + parsed_repo = urlparse.urlsplit(self.settings.repo) + repo_path = unicode(parsed_repo.path).rsplit(".git", 1)[0] self.check_url = self.settings.api_url + "repos" + repo_path + "/check-runs/" + self.settings.check_id + if parsed_repo.scheme == "https" and not parsed_repo.username: + new_netloc = "x-access-token:{}@{}".format(self.settings.token, parsed_repo.netloc) + parsed_repo = (parsed_repo.scheme, new_netloc, parsed_repo.path, parsed_repo.query, parsed_repo.fragment) + self.clone_url = urlparse.urlunsplit(parsed_repo) def code_review(self): self.reporter = self.reporter_factory() From 8fb1a3e51b3d2bb1edbd768642842c42ac4fa0ab Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Fri, 20 Sep 2019 16:04:03 +0300 Subject: [PATCH 23/32] feat(github): add inline comments for code_report --- _universum/modules/vcs/github_vcs.py | 68 +++++++++++++++------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/_universum/modules/vcs/github_vcs.py b/_universum/modules/vcs/github_vcs.py index d8a80554..e7cb9498 100644 --- a/_universum/modules/vcs/github_vcs.py +++ b/_universum/modules/vcs/github_vcs.py @@ -78,6 +78,16 @@ def __init__(self, *args, **kwargs): new_netloc = "x-access-token:{}@{}".format(self.settings.token, parsed_repo.netloc) parsed_repo = (parsed_repo.scheme, new_netloc, parsed_repo.path, parsed_repo.query, parsed_repo.fragment) self.clone_url = urlparse.urlunsplit(parsed_repo) + self.headers = { + "Accept": "application/vnd.github.antiope-preview+json", + "Authorization": "token " + self.settings.token + } + self.request = dict() + self.request["status"] = "in_progress" + self.request["output"] = { + "title": self.settings.check_name, + "summary": "" + } def code_review(self): self.reporter = self.reporter_factory() @@ -93,31 +103,32 @@ def get_review_link(self): def is_latest_version(self): return True + def _report(self): + result = requests.patch(self.check_url, json=self.request, headers=self.headers) + utils.check_request_result(result) + def code_report_to_review(self, report): - pass + # git show returns string, each file separated by \n, + # first line consists of commit id and commit comment, so it's skipped + commit_files = self.repo.git.show("--name-only", "--oneline", self.settings.checkout_id).split('\n')[1:] + comments = [] + for path, issues in report.iteritems(): + if path in commit_files: + for issue in issues: + comments.append(dict(path=path, + message=issue['message'], + start_line=issue['line'], + end_line=issue['line'], + annotation_level="warning")) + self.request["output"]["annotations"] = comments + self._report() def report_start(self, report_text): - headers = { - "Accept": "application/vnd.github.antiope-preview+json", - "Authorization": "token " + self.settings.token - } - request = { - "status": "in_progress", - "started_at": get_time(), - "output": { - "title": self.settings.check_name, - "summary": report_text - } - } - - result = requests.patch(self.check_url, json=request, headers=headers) - utils.check_request_result(result) + self.request["started_at"] = get_time() + self.request["output"]["summary"] = report_text + self._report() def report_result(self, result, report_text=None, no_vote=False): - headers = { - "Accept": "application/vnd.github.antiope-preview+json", - "Authorization": "token " + self.settings.token - } if result: conclusion = "success" else: @@ -126,15 +137,8 @@ def report_result(self, result, report_text=None, no_vote=False): if not report_text: report_text = "" - request = { - "status": "completed", - "completed_at": get_time(), - "conclusion": conclusion, - "output": { - "title": self.settings.check_name, - "summary": report_text - } - } - - result = requests.patch(self.check_url, json=request, headers=headers) - utils.check_request_result(result) + self.request["status"] = "completed" + self.request["completed_at"] = get_time() + self.request["conclusion"] = conclusion + self.request["output"]["summary"] = report_text + self._report() From 6fc65e2f87e61372aa675979b1cbe67b90ad3c1e Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Wed, 25 Sep 2019 14:50:12 +0300 Subject: [PATCH 24/32] fix(test): clean environment for tests, improve code style Without cleaning environment variables tests fail. The reason is that github variable coming from outer launch of the Universum affects work of the version of the Universum being tested. --- _universum/modules/vcs/github_vcs.py | 10 +++++----- tests/configs.py | 5 +++-- tests/test_report.py | 7 ++----- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/_universum/modules/vcs/github_vcs.py b/_universum/modules/vcs/github_vcs.py index e7cb9498..e7461497 100644 --- a/_universum/modules/vcs/github_vcs.py +++ b/_universum/modules/vcs/github_vcs.py @@ -1,8 +1,8 @@ # -*- coding: UTF-8 -*- import datetime -import requests import urlparse +import requests from ...lib.gravity import Dependency from ...lib import utils @@ -85,9 +85,9 @@ def __init__(self, *args, **kwargs): self.request = dict() self.request["status"] = "in_progress" self.request["output"] = { - "title": self.settings.check_name, - "summary": "" - } + "title": self.settings.check_name, + "summary": "" + } def code_review(self): self.reporter = self.reporter_factory() @@ -100,7 +100,7 @@ def update_review_version(self): def get_review_link(self): return self.settings.repo.rsplit(".git", 1)[0] + "/runs/" + self.settings.check_id - def is_latest_version(self): + def is_latest_version(self): # pylint: disable=no-self-use return True def _report(self): diff --git a/tests/configs.py b/tests/configs.py index 71bb6461..c7843382 100644 --- a/tests/configs.py +++ b/tests/configs.py @@ -1,10 +1,11 @@ +import os from _universum.configuration_support import Variations env_name = "virtual_universe" def run_virtual(cmd): - return ["bash", "-c", "source {}/bin/activate; {}".format(env_name, cmd)] + return ["env", "-i", "PATH=" + os.getenv("PATH"), "bash", "-c", "source {}/bin/activate; {}".format(env_name, cmd)] pylint_cmd = "universum_pylint --python-version 2 --rcfile pylintrc " + \ @@ -19,7 +20,7 @@ def run_virtual(cmd): dict(name="Install tests", artifacts="junit_results.xml", command=run_virtual("pip --default-timeout=1200 install .[test]")), dict(name="Make tests", artifacts="htmlcov", - command=run_virtual("PYTHONIOENCODING=utf-8 make test")), + command=run_virtual("export LANG=en_US.UTF-8; make test")), dict(name="Run static pylint", code_report=True, command=run_virtual("pip uninstall -y universum; " + pylint_cmd))]) diff --git a/tests/test_report.py b/tests/test_report.py index de0a0867..3cbfc73a 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,12 +1,11 @@ #!/usr/bin/env python # -*- coding: UTF-8 -*- -# pylint: disable = redefined-outer-name +# pylint: disable = redefined-outer-name, abstract-method -import httpretty import pytest import universum -import utils +from . import utils class ReportEnvironment(utils.TestEnvironment): @@ -34,7 +33,6 @@ def report_environment(tmpdir, git_client): yield ReportEnvironment(tmpdir, git_client) -# def test_github_run(stdout_checker, http_check, report_environment): def test_github_run(http_check, report_environment): http_check.assert_success_and_collect(universum.run, report_environment.settings, url=report_environment.path, method="PATCH") @@ -42,4 +40,3 @@ def test_github_run(http_check, report_environment): http_check.assert_request_body_contained("status", "in_progress") http_check.assert_request_body_contained("status", "completed") http_check.assert_request_body_contained("conclusion", "success") - # stdout_checker.assert_has_calls_with_param("==> Detected commit " + change) From cde7d51b4eb4a50d1a4a4106788dad8351013206 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Wed, 25 Sep 2019 16:32:32 +0300 Subject: [PATCH 25/32] feat(launch): change default Launcher output type to terminal for Jenkins builds --- _universum/modules/launcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_universum/modules/launcher.py b/_universum/modules/launcher.py index 16c104bc..cfb8180e 100644 --- a/_universum/modules/launcher.py +++ b/_universum/modules/launcher.py @@ -230,7 +230,7 @@ def define_arguments(argument_parser): help="Define whether to print build logs to console or store to vcs. " "Log file names are generated based on the names of build steps. " "Possible values: 'console', 'file'. By default, logs are printed to console " - "when the build is launched on TeamCity agent") + "when the build is launched on Jenkins or TeamCity agent") parser.add_argument("--launcher-config-path", "-lcp", dest="config_path", metavar="CONFIG_PATH", help="Project configs.py file location. Mandatory parameter") @@ -241,7 +241,7 @@ def __init__(self, *args, **kwargs): self.project_configs = None if self.settings.output is None: - if utils.detect_environment() != "tc": + if utils.detect_environment() == "terminal": self.settings.output = "file" else: self.settings.output = "console" From 524f0af47e5d363419ab4b969602f7446a2bbfbf Mon Sep 17 00:00:00 2001 From: miltolstoy <38533734+miltolstoy@users.noreply.github.com> Date: Thu, 26 Sep 2019 13:40:13 +0300 Subject: [PATCH 26/32] feat(jenkins_plugin): upload Jenkins plugin sources --- universum_log_collapser/pom.xml | 38 ++++ .../universum_log_collapser/Annotator.java | 165 ++++++++++++++++++ .../AnnotatorFactory.java | 13 ++ .../plugins/universum_log_collapser/Note.java | 19 ++ .../src/main/resources/index.jelly | 4 + .../universum_log_collapser/Note/script.js | 36 ++++ .../universum_log_collapser/Note/style.css | 35 ++++ 7 files changed, 310 insertions(+) create mode 100644 universum_log_collapser/pom.xml create mode 100644 universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Annotator.java create mode 100644 universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/AnnotatorFactory.java create mode 100644 universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Note.java create mode 100644 universum_log_collapser/src/main/resources/index.jelly create mode 100644 universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/script.js create mode 100644 universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/style.css diff --git a/universum_log_collapser/pom.xml b/universum_log_collapser/pom.xml new file mode 100644 index 00000000..d05c5f9d --- /dev/null +++ b/universum_log_collapser/pom.xml @@ -0,0 +1,38 @@ + + + 4.0.0 + + org.jenkins-ci.plugins + plugin + 3.4 + + + io.jenkins.plugins + universum_log_collapser + 1.0 + hpi + + 2.7.3 + 7 + + Universum Log Collapser + Collapsing universum-specific logs in Jenkins build console + + + MIT License + https://opensource.org/licenses/MIT + + + + + repo.jenkins-ci.org + https://repo.jenkins-ci.org/public/ + + + + + repo.jenkins-ci.org + https://repo.jenkins-ci.org/public/ + + + diff --git a/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Annotator.java b/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Annotator.java new file mode 100644 index 00000000..fd8e24d4 --- /dev/null +++ b/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Annotator.java @@ -0,0 +1,165 @@ +package io.jenkins.plugins.universum_log_collapser; + +import hudson.MarkupText; +import hudson.console.ConsoleAnnotator; + +import java.util.logging.*; +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import java.util.List; +import java.io.Serializable; +import java.util.ArrayList; + +class PaddingItem implements Serializable { + public int position; + public int spacesNumber; + + public PaddingItem(int p, int s) { + position = p; + spacesNumber = s; + } +} + +public class Annotator extends ConsoleAnnotator { + + private static final long serialVersionUID = 1L; + private static final Logger logger = Logger.getLogger(Annotator.class.getName()); + + private int labelsCnt = 1; + private boolean isIgnoredSection = false; + private List paddings = new ArrayList<>(); + private boolean universumLogActive = false; + + private Pattern sectionStartPattern = Pattern.compile("(^[|\\s]*)(\\d+\\.).*"); + private Pattern sectionEndPattern = Pattern.compile("^[|\\s]*└.*\\[.+\\].*"); + private Pattern sectionFailPattern = Pattern.compile("^[|\\s]*└.*\\[Failed\\].*"); + /* + "Reporting build result" section is showing summarized results of all + build steps. Each line of this section is treated as section start, but + have no content. That's why we are just collapsing all this section + content without content processing. + */ + private Pattern ignoredSectionPattern = Pattern.compile(".*Reporting build result.*"); + private Pattern universumLogStartPattern = Pattern.compile("^==> Universum \\d+\\.\\d+\\.\\d+ started execution$"); + private Pattern universumLogEndPattern = Pattern.compile("^==> Universum \\d+\\.\\d+\\.\\d+ finished execution$"); + private Pattern jenkinsLogEndPattern = Pattern.compile(".*Finished: .*"); + + public Annotator(Object context) {} + + /* + Before: + 1. Preparing repository + | ==> Adding file /var/lib/jenkins/workspace/universum_job/artifacts/REPOSITORY_STATE.txt to artifacts... + | 1.1. Copying sources to working directory + | | ==> Moving sources to '/var/lib/jenkins/workspace/universum_job/temp'... + | └ [Success] + | + └ [Success] + + After: + + +
+ | ==> Adding file /var/lib/jenkins/workspace/universum_job/artifacts/REPOSITORY_STATE.txt to artifacts... + + +
+ | | ==> Moving sources to '/var/lib/jenkins/workspace/universum_job/temp'... + | └ [Success] +
+ + + | + └ [Success] +
+ + + */ + @Override + public Annotator annotate(Object context, MarkupText text) { + String textStr = text.toString(true); + logger.info(textStr); + + universumLogActive = universumLogActive || universumLogStartPattern.matcher(textStr).find(); + if (!universumLogActive) { + logger.info("Skip non-universum log"); + return this; + } + + if (universumLogEndPattern.matcher(textStr).find()) { + universumLogActive = false; + } + + for (PaddingItem p : paddings) { + text.addMarkup(p.position, ""); + } + + Matcher sectionStartMatcher = sectionStartPattern.matcher(textStr); + if (sectionStartMatcher.find()) { + processSectionStart(text, sectionStartMatcher); + return this; + } + + Matcher sectionEndMatcher = sectionEndPattern.matcher(textStr); + if (sectionEndMatcher.find()) { + processSectionEnd(text, sectionFailPattern.matcher(textStr)); + return this; + } + + Matcher jenkinsLogEndMatcher = jenkinsLogEndPattern.matcher(textStr); + if (jenkinsLogEndMatcher.find()) { + text.addMarkup(text.length(), ""); + return this; + } + + return this; + } + + + private void processSectionStart(MarkupText text, Matcher sectionStartMatcher) { + if (isIgnoredSection) { + logger.info("Skip ignored section"); + return; + } + + logger.info("Section start found"); + if (ignoredSectionPattern.matcher(text.toString(true)).find()) { + isIgnoredSection = true; + } + + int sectionNumberStartPosition = sectionStartMatcher.end(1); + int sectionNumberEndPosition = sectionStartMatcher.end(2); + paddings.add(new PaddingItem(sectionNumberStartPosition, + sectionNumberEndPosition - sectionNumberStartPosition)); + + String inputId = "hide-block-" + labelsCnt++; + text.addMarkup(0, "
"); + } + + private void processSectionEnd(MarkupText text, Matcher sectionFailMatcher) { + logger.info("Section end found"); + paddings.remove(paddings.size() - 1); + + if (sectionFailMatcher.find()) { + logger.info("Failed section found"); + text.addMarkup(0, ""); + text.addMarkup(text.length(), ""); + } + text.addMarkup(text.length(), "
"); + + isIgnoredSection = false; + } +} diff --git a/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/AnnotatorFactory.java b/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/AnnotatorFactory.java new file mode 100644 index 00000000..74b5a2c7 --- /dev/null +++ b/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/AnnotatorFactory.java @@ -0,0 +1,13 @@ +package io.jenkins.plugins.universum_log_collapser; + +import hudson.Extension; +import hudson.console.ConsoleAnnotatorFactory; + +// Plugin entry point +@Extension +public class AnnotatorFactory extends ConsoleAnnotatorFactory { + @Override + public Annotator newInstance(T context) { + return new Annotator(context); + } +} diff --git a/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Note.java b/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Note.java new file mode 100644 index 00000000..4d42ac3f --- /dev/null +++ b/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Note.java @@ -0,0 +1,19 @@ +package io.jenkins.plugins.universum_log_collapser; + +import hudson.Extension; +import hudson.MarkupText; +import hudson.console.ConsoleAnnotationDescriptor; +import hudson.console.ConsoleNote; + +public class Note extends ConsoleNote { + + private static final long serialVersionUID = 1L; + + public Annotator annotate(T context, MarkupText text, int charPos) { + return null; + } + + // Extension point to connect plugin style.css to Console Ouput HTML page + @Extension + public static final class Descriptor extends ConsoleAnnotationDescriptor {} +} diff --git a/universum_log_collapser/src/main/resources/index.jelly b/universum_log_collapser/src/main/resources/index.jelly new file mode 100644 index 00000000..6bb5a210 --- /dev/null +++ b/universum_log_collapser/src/main/resources/index.jelly @@ -0,0 +1,4 @@ + +
+ Collapsing universum-specific logs in Jenkins build console +
diff --git a/universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/script.js b/universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/script.js new file mode 100644 index 00000000..40c7a73f --- /dev/null +++ b/universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/script.js @@ -0,0 +1,36 @@ +var coloringTimeoutMilliSeconds = 250; +var timerId = setInterval(colorFailedSections, coloringTimeoutMilliSeconds); + +function colorLblsAscendant(el) { + var sectionLbl = el.getElementsByClassName("sectionLbl")[0]; + sectionLbl.style.color = "red"; + sectionLbl.getElementsByTagName("span")[0].style.cssText = "color:red !important"; + var parent = el.parentNode; + if (parent.className != "console-output") { + colorLblsAscendant(parent.previousSibling); + } +} + +function colorFailedSections() { + var results = document.getElementsByClassName("failed_result"); + for (var i = 0; i < results.length; i++) { + var element = results[i]; + colorLblsAscendant(element.parentNode.previousSibling); + element.className = "failed_result_handled"; + } +} + +function finishColoring() { + clearInterval(timerId); + + /* + If universum step execution takes long time, Jenkins closes current
 and opens new one, which broke
+     sections displaying. This is fixed by page refresh.
+     */
+    var preElement = document.getElementsByClassName("console-output")[0];
+    if (preElement.getElementsByTagName("pre").length > 0) {
+        document.location.reload();
+    }
+
+    setTimeout(colorFailedSections, coloringTimeoutMilliSeconds);
+}
diff --git a/universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/style.css b/universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/style.css
new file mode 100644
index 00000000..8f9f0a39
--- /dev/null
+++ b/universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/style.css
@@ -0,0 +1,35 @@
+.sectionLbl {
+    color: black;
+}
+.sectionLbl span {
+    color: black !important;
+}
+
+.hide {
+    display: none; 
+}
+.hide + label ~ div {
+    display: none;
+}
+.hide + label {
+    color: black;
+    cursor: pointer;
+    display: inline-block; 
+}
+.hide:checked + label + div {
+    display: block; 
+}
+
+.hide + label + div + .nl {
+    display: block;
+}
+.hide:checked + label + div + .nl::after {
+    display: none;
+}
+
+.hide + label .sectionLbl::before {
+    content: "[+] ";
+}
+.hide:checked + label .sectionLbl::before {
+    content: "[-] ";
+}

From d18bf6ddd3ec0c0e8c2094346667245a0152c5d7 Mon Sep 17 00:00:00 2001
From: miltolstoy <38533734+miltolstoy@users.noreply.github.com>
Date: Mon, 30 Sep 2019 15:56:12 +0300
Subject: [PATCH 27/32] fix(jenkins_plugin): logic to update page content on
 long steps executing

---
 .../universum_log_collapser/Note/script.js    | 147 +++++++++++++++++-
 1 file changed, 140 insertions(+), 7 deletions(-)

diff --git a/universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/script.js b/universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/script.js
index 40c7a73f..b60f38ec 100644
--- a/universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/script.js
+++ b/universum_log_collapser/src/main/resources/io/jenkins/plugins/universum_log_collapser/Note/script.js
@@ -2,6 +2,9 @@ var coloringTimeoutMilliSeconds = 250;
 var timerId = setInterval(colorFailedSections, coloringTimeoutMilliSeconds);
 
 function colorLblsAscendant(el) {
+    if (el == null) {
+        return;
+    }
     var sectionLbl = el.getElementsByClassName("sectionLbl")[0];
     sectionLbl.style.color = "red";
     sectionLbl.getElementsByTagName("span")[0].style.cssText = "color:red !important";
@@ -12,6 +15,8 @@ function colorLblsAscendant(el) {
 }
 
 function colorFailedSections() {
+    fetchNext = myFetchNext;
+
     var results = document.getElementsByClassName("failed_result");
     for (var i = 0; i < results.length; i++) {
         var element = results[i];
@@ -22,15 +27,143 @@ function colorFailedSections() {
 
 function finishColoring() {
     clearInterval(timerId);
+    setTimeout(colorFailedSections, coloringTimeoutMilliSeconds);
+}
+
+/*
+Overrided implementation of Jenkins fetchNext() function.
+It's called when Universum step takes long time and console output is splitted to several 
 tags instead of one.
+e: global 
 tag
+href: URL to send request for further log
+text: further log, received from URL
+*/
+var myFetchNext = function(e, href) {
+    var headers = {};
+    if (e.consoleAnnotator != undefined) {
+        headers["X-ConsoleAnnotator"] = e.consoleAnnotator;
+    }
+
+	new Ajax.Request(href, {method: "post", parameters: {"start":e.fetchedBytes},
+        requestHeaders: headers, onComplete: function(rsp,_) {
+        var stickToBottom = scroller.isSticking();
+        var text = rsp.responseText;
+        if(text != "") {
+            /*
+            Following code was replaced by customTextHandle() call:
+            var p = document.createElement("DIV");
+            e.appendChild(p);
+            if (p.outerHTML) {
+                p.outerHTML = '
' + text + '
'; + p = e.lastChild; + } else { + p.innerHTML = text; + } + Behaviour.applySubtree(p); + */ + customTextHandle(e, text); + ElementResizeTracker.fireResizeCheck(); + if(stickToBottom) { + scroller.scrollToBottom(); + } + } + + e.fetchedBytes = rsp.getResponseHeader("X-Text-Size"); + e.consoleAnnotator = rsp.getResponseHeader("X-ConsoleAnnotator"); + if(rsp.getResponseHeader("X-More-Data") == "true") { + setTimeout(function() {fetchNext(e,href);}, 1000); + } else { + $("spinner").style.display = "none"; + } + } + }); +} + +/* +Find last closed tags in text, which already exists on page +For each closed tag: + - find tag element on page + - find closed tag position in received text + - append received tag content to existing tag element +*/ +function customTextHandle(e, text) { + var lastClosedTags = e.innerHTML.match(new RegExp("(<\/\\w+>)+$", "g")); // e.g.
+ var closedTagsArr = lastClosedTags[0].match(new RegExp("<\/\\w+>", "g")); // split to array /* - If universum step execution takes long time, Jenkins closes current
 and opens new one, which broke
-     sections displaying. This is fixed by page refresh.
-     */
-    var preElement = document.getElementsByClassName("console-output")[0];
-    if (preElement.getElementsByTagName("pre").length > 0) {
-        document.location.reload();
+    Key - tag name. Value - element object.
+    Used to find proper element to append a text in case of nested tags with the same name.
+    */
+    var tagsElementMap = new Map();
+    var currTextPosition = 0;
+    for (var i = 0; i < closedTagsArr.length; i++) {
+        var tagName = closedTagsArr[i].match(new RegExp("<\/(\\w+)>"))[1]; // "" or "
" => "div" + var tagElements = e.getElementsByTagName(tagName); + var tagPosition = findClosedTagPosition(text, tagName, currTextPosition); + + var element = null; + var closedElement = tagsElementMap.get(tagName); + if (closedElement != undefined) { + // if we already handled one
, get its parent + element = closedElement.parentElement; + } else { + // just get last element with this tag + element = tagElements[tagElements.length - 1]; + } + + // if closing tag was not found in text + if (tagPosition == 0) { + // just append remaining text and exit + element.innerHTML += text.substr(currTextPosition); + break; + } + + // append needed part of text to tag element content + var symbolsCount = tagPosition - currTextPosition; + element.innerHTML += text.substr(currTextPosition, symbolsCount); + currTextPosition += symbolsCount; + tagsElementMap.set(tagName, element); } +} - setTimeout(colorFailedSections, coloringTimeoutMilliSeconds); +/* +Find first closed tag in text, which: + 1. was not opened in text + 2. which position > lastInsertPosition + +lastInsertPosition = 0 +bla
blabla
blablabla + ^^^^^^ + needed tag + +lastInsertPosition = 3 +bla
blabla
blablabla + ^^^^^^ + needed tag +*/ +function findClosedTagPosition(text, tagName, lastInsertPosition) { + var regexp = new RegExp("<\/" + tagName + ">|<" + tagName + ">", "g"); // opened or closed tag + var tagPosition = 0; + var tagOpenCloseCounter = 0; // to find first closed, but not opened tag + while (true) { + var found = regexp.exec(text); + if (found === null) { // end of text + break; + } + if (regexp.lastIndex <= lastInsertPosition) { + continue; + } + + if (found[0].includes("/")) { + tagOpenCloseCounter -= 1; // closing tag found + } else { + tagOpenCloseCounter += 1; // opening tag found + } + + if (tagOpenCloseCounter < 0) { // closing tags quantity is bigger than opening tags quantity + tagPosition = regexp.lastIndex; + break; + } + } + + return tagPosition; } From c9b3461aa33df9eeeaa8a2d3b94742308663b805 Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Tue, 1 Oct 2019 20:05:45 +0300 Subject: [PATCH 28/32] fix(p4): pass only expected exceptions on file revert --- _universum/modules/vcs/perforce_vcs.py | 13 +++++--- tests/perforce_utils.py | 3 +- tests/test_p4_revert_unshelved.py | 45 +++++++++++++++++++++++++- tests/utils.py | 11 ++++--- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/_universum/modules/vcs/perforce_vcs.py b/_universum/modules/vcs/perforce_vcs.py index 9c331081..92666ef9 100644 --- a/_universum/modules/vcs/perforce_vcs.py +++ b/_universum/modules/vcs/perforce_vcs.py @@ -555,15 +555,20 @@ def prepare_repository(self): self.swarm.mappings_dict = self.mappings_dict @make_block("Cleaning workspace", pass_errors=False) - @catch_p4exception(ignore_if="doesn't exist") def clean_workspace(self): try: self.p4.client = self.client_name report = self.p4.run_revert("//...") self.p4report(report) - except P4Exception: - pass - self.p4.delete_client(self.client_name) + except P4Exception as e: + if "Client '{}' unknown".format(self.client_name) not in e.value \ + and "file(s) not opened on this client" not in e.value: + self.structure.fail_current_block(e.value) + try: + self.p4.delete_client(self.client_name) + except P4Exception as e: + if "Client '{}' doesn't exist".format(self.client_name) not in e.value: + self.structure.fail_current_block(e.value) def finalize(self): with Uninterruptible(self.out.log_exception) as run: diff --git a/tests/perforce_utils.py b/tests/perforce_utils.py index ccf04bd0..0b174602 100644 --- a/tests/perforce_utils.py +++ b/tests/perforce_utils.py @@ -218,6 +218,7 @@ def __init__(self, perforce_workspace, directory, test_type): self.nonwritable_file = perforce_workspace.nonwritable_file self.p4 = perforce_workspace.p4 self.depot = perforce_workspace.depot + self.client_name = "p4_disposable_workspace" super(P4Environment, self).__init__(directory, test_type) self.settings.Vcs.type = "p4" @@ -225,7 +226,7 @@ def __init__(self, perforce_workspace, directory, test_type): self.settings.PerforceVcs.user = perforce_workspace.p4.user self.settings.PerforceVcs.password = perforce_workspace.p4.password try: - self.settings.PerforceMainVcs.client = "p4_disposable_workspace" + self.settings.PerforceMainVcs.client = self.client_name self.settings.PerforceMainVcs.force_clean = True except AttributeError: pass diff --git a/tests/test_p4_revert_unshelved.py b/tests/test_p4_revert_unshelved.py index 1504ba4c..721e6fb6 100644 --- a/tests/test_p4_revert_unshelved.py +++ b/tests/test_p4_revert_unshelved.py @@ -4,8 +4,10 @@ import os import pytest +import universum from _universum.lib.gravity import construct_component -from _universum.modules.vcs import perforce_vcs, vcs +from _universum.modules.vcs import perforce_vcs +from .perforce_utils import P4Environment from . import utils @@ -131,3 +133,44 @@ def create_file(filename): for result, expected in zip(diff, expected_path): assert result == expected + + +@pytest.fixture() +def perforce_environment(perforce_workspace, tmpdir): + yield P4Environment(perforce_workspace, tmpdir, test_type="main") + + +def test_p4_error_revert(perforce_environment, stdout_checker, capsys): + p4 = perforce_environment.p4 + p4_file = perforce_environment.repo_file + + config = """ +from _universum.configuration_support import Variations + +configs = Variations([dict(name="Restrict changes", command=["chmod", "-R", "555", "."]), + dict(name="Check", command=["ls", "-la"])]) +""" + p4.run_edit(perforce_environment.depot) + p4_file.write(config) + change = p4.fetch_change() + change["Description"] = "CL for shelving" + shelve_cl = p4.save_change(change)[0].split()[1] + p4.run_shelve("-fc", shelve_cl) + + settings = perforce_environment.settings + settings.Launcher.output = "console" + settings.PerforceMainVcs.shelve_cls = [shelve_cl] + settings.Launcher.config_path = p4_file.basename + + result = universum.run(settings) + # Clean up the directory at once to make sure it doesn't remain non-writable even if some assert fails + perforce_environment.temp_dir.chmod(0777, rec=1) + perforce_environment.temp_dir.remove(rec=1) + + assert result != 0 + # The following checks make sure all following actions were triggered despite unsuccessful: + # full revert, client deleting and sources clean up + stdout_checker.assert_has_calls_with_param("Errors during command execution( \"p4 revert //...\" )") + stdout_checker.assert_has_calls_with_param( + "Errors during command execution( \"p4 client -d {}\" )".format(perforce_environment.client_name)) + assert "[Errno 13] Permission denied" in capsys.readouterr().err diff --git a/tests/utils.py b/tests/utils.py index 54260ca8..a1342c5b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -81,25 +81,26 @@ def create_empty_settings(test_type): class TestEnvironment(object): - def __init__(self, directory, test_type): + def __init__(self, temp_dir, test_type): + self.temp_dir = temp_dir self.settings = create_empty_settings(test_type) if test_type == "poll": self.settings.Poll.db_file = self.db_file self.settings.JenkinsServerForTrigger.trigger_url = "https://localhost/?cl=%s" self.settings.AutomationServer.type = "jenkins" - self.settings.ProjectDirectory.project_root = unicode(directory.mkdir("project_root")) + self.settings.ProjectDirectory.project_root = unicode(self.temp_dir.mkdir("project_root")) elif test_type == "submit": self.settings.Submit.commit_message = "Test CL" # For submitter, the main working dir (project_root) should be the root # of the VCS workspace/client self.settings.ProjectDirectory.project_root = unicode(self.vcs_cooking_dir) elif test_type == "main": - configs_file = directory.join("configs.py") + configs_file = self.temp_dir.join("configs.py") configs_file.write(simple_test_config) self.settings.Launcher.config_path = unicode(configs_file) - self.settings.ArtifactCollector.artifact_dir = unicode(directory.mkdir("artifacts")) + self.settings.ArtifactCollector.artifact_dir = unicode(self.temp_dir.mkdir("artifacts")) # The project_root directory must not exist before launching main - self.settings.ProjectDirectory.project_root = unicode(directory.join("project_root")) + self.settings.ProjectDirectory.project_root = unicode(self.temp_dir.join("project_root")) self.settings.Output.type = "term" From da0e733d9c15c27423dc10929391aade1481957f Mon Sep 17 00:00:00 2001 From: miltolstoy <38533734+miltolstoy@users.noreply.github.com> Date: Wed, 2 Oct 2019 09:06:07 +0300 Subject: [PATCH 29/32] feat(jenkins_plugin): increase version, update gitignore with plugin-specific files --- universum_log_collapser/.gitignore | 5 +++++ universum_log_collapser/pom.xml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 universum_log_collapser/.gitignore diff --git a/universum_log_collapser/.gitignore b/universum_log_collapser/.gitignore new file mode 100644 index 00000000..c32b7144 --- /dev/null +++ b/universum_log_collapser/.gitignore @@ -0,0 +1,5 @@ +.classpath +.factorypath +.project +.settings +target diff --git a/universum_log_collapser/pom.xml b/universum_log_collapser/pom.xml index d05c5f9d..eec1a057 100644 --- a/universum_log_collapser/pom.xml +++ b/universum_log_collapser/pom.xml @@ -9,7 +9,7 @@ io.jenkins.plugins universum_log_collapser - 1.0 + 1.1 hpi 2.7.3 From 9976350a46703160fce735a247025d06fe9436aa Mon Sep 17 00:00:00 2001 From: miltolstoy <38533734+miltolstoy@users.noreply.github.com> Date: Tue, 8 Oct 2019 15:22:22 +0300 Subject: [PATCH 30/32] fix(jenkins_plugin): fix log displaying at job terminating --- universum_log_collapser/pom.xml | 2 +- .../jenkins/plugins/universum_log_collapser/Annotator.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/universum_log_collapser/pom.xml b/universum_log_collapser/pom.xml index eec1a057..740fd792 100644 --- a/universum_log_collapser/pom.xml +++ b/universum_log_collapser/pom.xml @@ -9,7 +9,7 @@ io.jenkins.plugins universum_log_collapser - 1.1 + 1.2 hpi 2.7.3 diff --git a/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Annotator.java b/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Annotator.java index fd8e24d4..cd372992 100644 --- a/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Annotator.java +++ b/universum_log_collapser/src/main/java/io/jenkins/plugins/universum_log_collapser/Annotator.java @@ -43,6 +43,7 @@ public class Annotator extends ConsoleAnnotator { private Pattern universumLogStartPattern = Pattern.compile("^==> Universum \\d+\\.\\d+\\.\\d+ started execution$"); private Pattern universumLogEndPattern = Pattern.compile("^==> Universum \\d+\\.\\d+\\.\\d+ finished execution$"); private Pattern jenkinsLogEndPattern = Pattern.compile(".*Finished: .*"); + private Pattern healthyLogPattern = Pattern.compile("^\\s+[\\|└]\\s+.*"); public Annotator(Object context) {} @@ -100,6 +101,11 @@ public Annotator annotate(Object context, MarkupText text) { } for (PaddingItem p : paddings) { + if (!healthyLogPattern.matcher(textStr).find()) { + logger.info("Log is broken, identation expected"); + universumLogActive = false; + return this; + } text.addMarkup(p.position, ""); } From 271aacede0bddce3bdb53710da3e1975c9fdd92d Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan <46348880+k-dovgan@users.noreply.github.com> Date: Tue, 8 Oct 2019 17:29:27 +0300 Subject: [PATCH 31/32] fix(universum): handle SIGTERM properly --- tests/test_integration.py | 33 +++++++++++++++++++++++++++++++++ universum.py | 13 +++---------- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index b2783582..d8371ca9 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -2,6 +2,11 @@ # -*- coding: UTF-8 -*- import os +import signal +import time + +import pytest +import sh def get_line_with_text(text, log): @@ -319,3 +324,31 @@ def test_environment(universum_runner): configs = upper * lower """) assert "This string should be in log" in log + + +@pytest.mark.parametrize("terminate_type", [signal.SIGINT, signal.SIGTERM], ids=["interrupt", "terminate"]) +def test_abort(local_sources, tmpdir, terminate_type): + config = """ +from _universum.configuration_support import Variations + +configs = Variations([dict(name="Long step", command=["sleep", "10"])]) * 5 +""" + config_file = tmpdir.join("configs.py") + config_file.write(config) + + cmd = sh.Command(os.path.join(os.getcwd(), "universum.py")) + + def handle_out(line): + print line.rstrip() + + process = cmd(*(["-lo", "console", "-vt", "none", + "-pr", unicode(tmpdir.join("project_root")), + "-ad", unicode(tmpdir.join("artifacts")), + "-fsd", unicode(local_sources.root_directory), + "-lcp", unicode(config_file)]), + _iter=True, _bg_exc=False, _bg=True, _out=handle_out, _err=handle_out) + time.sleep(5) + process.signal(terminate_type) + with pytest.raises(sh.ErrorReturnCode): + process.wait() + assert process.exit_code == 3 diff --git a/universum.py b/universum.py index 39b48311..170c49b7 100755 --- a/universum.py +++ b/universum.py @@ -43,17 +43,15 @@ def run(settings): main_module = construct_component(settings.main_class, settings) main_module.out.log("{} {} started execution".format(__title__, __version__)) - finalized = False + def signal_handler(signal_number, stack_frame): + raise KeyboardInterrupt - def finalize(): - if not finalized: - main_module.finalize() + signal.signal(signal.SIGTERM, signal_handler) try: with Uninterruptible(main_module.out.log_exception) as run_function: run_function(main_module.execute) run_function(main_module.finalize) - finalized = True except SilentAbortException as e: result = e.application_exit_code @@ -64,11 +62,6 @@ def finalize(): main_module.out.report_build_problem("Unexpected error while executing script.") result = 2 - atexit.register(finalize) - signal.signal(signal.SIGTERM, finalize) - signal.signal(signal.SIGHUP, finalize) - signal.signal(signal.SIGINT, finalize) - main_module.out.log("{} {} finished execution".format(__title__, __version__)) return result From 4853b09119c67e635c076ddea53037d110c09f8a Mon Sep 17 00:00:00 2001 From: Kateryna Dovgan Date: Wed, 9 Oct 2019 14:46:12 +0300 Subject: [PATCH 32/32] update(docs): update change log to Universum 0.18.2 --- CHANGELOG.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 42a34b65..37958ffa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,27 @@ Change log ========== +0.18.2 (2019-10-09) +------------------- + +New features +~~~~~~~~~~~~ + +* **vcs:** add 'github' VCS type +* **vcs:** implement GitHub as code review system +* **github:** add inline comments for code_report +* **jenkins_plugin:** add jenkins plugin for Universum logs pretty printing +* **test:** clean environment for tests + +Bug fixes +~~~~~~~~~ + +* **handle SIGTERM properly** +* **p4:** ignore only expected exceptions on file revert +* **test:** single poll fails because httpretty conflicts with docker-py +* **test:** whitespaces in local paths + + 0.18.1 (2019-07-23) -------------------