Universum 0.18.2

CHANGELOG.rst

@@ -1,6 +1,27 @@
 Change log
 ==========
 
+0.18.2 (2019-10-09)
+-------------------
+
+New features
+~~~~~~~~~~~~
+
+* **vcs:** add 'github' VCS type
+* **vcs:** implement GitHub as code review system
+* **github:** add inline comments for code_report
+* **jenkins_plugin:** add jenkins plugin for Universum logs pretty printing
+* **test:** clean environment for tests
+
+Bug fixes
+~~~~~~~~~
+
+* **handle SIGTERM properly**
+* **p4:** ignore only expected exceptions on file revert
+* **test:** single poll fails because httpretty conflicts with docker-py
+* **test:** whitespaces in local paths
+
+
 0.18.1 (2019-07-23)
 -------------------
 

_universum/__init__.py

@@ -1,4 +1,4 @@
 # -*- coding: UTF-8 -*-
 
 __title__ = "Universum"
-__version__ = "0.18.1"
+__version__ = "0.18.2"

_universum/lib/utils.py

@@ -7,6 +7,7 @@
 import sys
 import traceback
 
+from _universum.lib.ci_exception import CiException
 from .ci_exception import CriticalCiException, SilentAbortException
 from .module_arguments import IncorrectParameterError
 
@@ -24,7 +25,8 @@
     "convert_to_str",
     "unify_argument_list",
     "Uninterruptible",
-    "make_block"
+    "make_block",
+    "check_request_result"
 ]
 
 # For proper unicode symbols processing
@@ -208,3 +210,10 @@ def function_in_block(self, *args, **kwargs):
             return self.structure.run_in_block(func, block_name, pass_errors, self, *args, **kwargs)
         return function_in_block
     return decorated_function
+
+
+def check_request_result(result):
+    if result.status_code != 200:
+        text = "Invalid return code " + unicode(result.status_code) + ". Response is:\n"
+        text += result.text
+        raise CiException(text)

_universum/modules/launcher.py

@@ -230,7 +230,7 @@ def define_arguments(argument_parser):
                             help="Define whether to print build logs to console or store to vcs. "
                                  "Log file names are generated based on the names of build steps. "
                                  "Possible values: 'console', 'file'. By default, logs are printed to console "
-                                 "when the build is launched on TeamCity agent")
+                                 "when the build is launched on Jenkins or TeamCity agent")
 
         parser.add_argument("--launcher-config-path", "-lcp", dest="config_path", metavar="CONFIG_PATH",
                             help="Project configs.py file location. Mandatory parameter")
@@ -241,7 +241,7 @@ def __init__(self, *args, **kwargs):
         self.project_configs = None
 
         if self.settings.output is None:
-            if utils.detect_environment() != "tc":
+            if utils.detect_environment() == "terminal":
                 self.settings.output = "file"
             else:
                 self.settings.output = "console"

_universum/modules/vcs/git_vcs.py

@@ -70,6 +70,7 @@ def __init__(self, out, *args, **kwargs):
             def line_dropped(self, line):
                 self.out.log(line)
 
+        self.clone_url = self.settings.repo
         self.repo = None
         self.logger = Progress(self.out)
 
@@ -84,16 +85,16 @@ def line_dropped(self, line):
     @make_block("Cloning repository")
     @catch_git_exception()
     def clone_and_fetch(self, history_depth=None):
-        self.out.log("Cloning '" + self.settings.repo + "'...")
+        self.out.log("Cloning '" + self.clone_url + "'...")
         destination_directory = convert_to_str(self.settings.project_root)
         if history_depth:
-            self.repo = git.Repo.clone_from(self.settings.repo, destination_directory,
+            self.repo = git.Repo.clone_from(self.clone_url, destination_directory,
                                             depth=history_depth, no_single_branch=True, progress=self.logger)
         else:
-            self.repo = git.Repo.clone_from(self.settings.repo, destination_directory, progress=self.logger)
+            self.repo = git.Repo.clone_from(self.clone_url, destination_directory, progress=self.logger)
 
         self.sources_need_cleaning = True
-        self.append_repo_status("Git repo: " + self.settings.repo + "\n\n")
+        self.append_repo_status("Git repo: " + self.clone_url + "\n\n")
 
         self.out.log("Please note that default remote name is 'origin'")
         if self.settings.refspec:

_universum/modules/vcs/github_vcs.py

@@ -0,0 +1,144 @@
+# -*- coding: UTF-8 -*-
+
+import datetime
+import urlparse
+import requests
+
+from ...lib.gravity import Dependency
+from ...lib import utils
+from ..reporter import ReportObserver, Reporter
+from . import git_vcs
+
+__all__ = [
+    "GithubMainVcs"
+]
+
+
+def get_time():
+    return datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
+
+
+class GithubMainVcs(ReportObserver, git_vcs.GitMainVcs):
+    """
+    This class mostly contains functions for Gihub report observer
+    """
+    reporter_factory = Dependency(Reporter)
+
+    @staticmethod
+    def define_arguments(argument_parser):
+        parser = argument_parser.get_or_create_group("GitHub", "GitHub repository settings")
+
+        parser.add_argument("--github-token", "-ght", dest="token", metavar="GITHUB_TOKEN",
+                            help="GitHub API token; for details see "
+                                 "https://developer.github.com/v3/oauth_authorizations/")
+        parser.add_argument("--github-check-name", "-ghc", dest="check_name", metavar="GITHUB_CHECK_NAME",
+                            default="Universum check", help="The name of Github check run")
+        parser.add_argument("--github-check-id", "-ghi", dest="check_id", metavar="GITHUB_CHECK_ID",
+                            help="Github check run ID")
+        parser.add_argument("--github-api-url", "-gha", dest="api_url", metavar="GITHUB_API_URL",
+                            default="https://api.github.com/", help="API URL for integration")
+
+    def __init__(self, *args, **kwargs):
+        super(GithubMainVcs, self).__init__(*args, **kwargs)
+        self.reporter = None
+        utils.check_required_option(self.settings, "checkout_id", """
+                    git checkout id for github is not specified.
+
+                    For github the git checkout id defines the commit to be checked and reported.
+                    Please specify the checkout id by using '--git-checkout-id' ('-gco')
+                    command line parameter or by setting GIT_CHECKOUT_ID environment variable.
+
+                    In CI builds commit ID is usually extracted from webhook and handled automatically.  
+                """)
+        utils.check_required_option(self.settings, "token", """
+                    github api token is not specified.
+
+                    GitHub API token is used for application authorization. Expected format is
+                    'v1.4e96f8c2d1922c3b154a65ca7ecb91f6994fb0c5'.
+                    Please specify the token by using '--github-token' ('-ght')
+                    command line parameter or by setting GITHUB_TOKEN environment variable.
+
+                    In CI builds github api token is expected to be acquired automatically before build.
+                    For information on how to acquire the token please see
+                    https://developer.github.com/v3/oauth_authorizations/
+                """)
+        utils.check_required_option(self.settings, "check_id", """
+                    github check id is not specified.
+
+                    GitHub check runs each have unique ID, that is used to update check result.
+                    To integrate Universum with GitHub, check run should be already created before performing
+                    actual check. Please specify check run ID by using '--github-check-id' ('-ghi')
+                    command line parameter or by setting GITHUB_CHECK_ID environment variable.
+                """)
+
+        parsed_repo = urlparse.urlsplit(self.settings.repo)
+        repo_path = unicode(parsed_repo.path).rsplit(".git", 1)[0]
+        self.check_url = self.settings.api_url + "repos" + repo_path + "/check-runs/" + self.settings.check_id
+        if parsed_repo.scheme == "https" and not parsed_repo.username:
+            new_netloc = "x-access-token:{}@{}".format(self.settings.token, parsed_repo.netloc)
+            parsed_repo = (parsed_repo.scheme, new_netloc, parsed_repo.path, parsed_repo.query, parsed_repo.fragment)
+        self.clone_url = urlparse.urlunsplit(parsed_repo)
+        self.headers = {
+            "Accept": "application/vnd.github.antiope-preview+json",
+            "Authorization": "token " + self.settings.token
+        }
+        self.request = dict()
+        self.request["status"] = "in_progress"
+        self.request["output"] = {
+            "title": self.settings.check_name,
+            "summary": ""
+        }
+
+    def code_review(self):
+        self.reporter = self.reporter_factory()
+        self.reporter.subscribe(self)
+        return self
+
+    def update_review_version(self):
+        self.out.log("GitHub has no review versions")
+
+    def get_review_link(self):
+        return self.settings.repo.rsplit(".git", 1)[0] + "/runs/" + self.settings.check_id
+
+    def is_latest_version(self):  # pylint: disable=no-self-use
+        return True
+
+    def _report(self):
+        result = requests.patch(self.check_url, json=self.request, headers=self.headers)
+        utils.check_request_result(result)
+
+    def code_report_to_review(self, report):
+        # git show returns string, each file separated by \n,
+        # first line consists of commit id and commit comment, so it's skipped
+        commit_files = self.repo.git.show("--name-only", "--oneline", self.settings.checkout_id).split('\n')[1:]
+        comments = []
+        for path, issues in report.iteritems():
+            if path in commit_files:
+                for issue in issues:
+                    comments.append(dict(path=path,
+                                         message=issue['message'],
+                                         start_line=issue['line'],
+                                         end_line=issue['line'],
+                                         annotation_level="warning"))
+        self.request["output"]["annotations"] = comments
+        self._report()
+
+    def report_start(self, report_text):
+        self.request["started_at"] = get_time()
+        self.request["output"]["summary"] = report_text
+        self._report()
+
+    def report_result(self, result, report_text=None, no_vote=False):
+        if result:
+            conclusion = "success"
+        else:
+            conclusion = "failure"
+
+        if not report_text:
+            report_text = ""
+
+        self.request["status"] = "completed"
+        self.request["completed_at"] = get_time()
+        self.request["conclusion"] = conclusion
+        self.request["output"]["summary"] = report_text
+        self._report()

_universum/modules/vcs/perforce_vcs.py

@@ -555,15 +555,20 @@ def prepare_repository(self):
             self.swarm.mappings_dict = self.mappings_dict
 
     @make_block("Cleaning workspace", pass_errors=False)
-    @catch_p4exception(ignore_if="doesn't exist")
     def clean_workspace(self):
         try:
             self.p4.client = self.client_name
             report = self.p4.run_revert("//...")
             self.p4report(report)
-        except P4Exception:
-            pass
-        self.p4.delete_client(self.client_name)
+        except P4Exception as e:
+            if "Client '{}' unknown".format(self.client_name) not in e.value \
+                    and "file(s) not opened on this client" not in e.value:
+                self.structure.fail_current_block(e.value)
+        try:
+            self.p4.delete_client(self.client_name)
+        except P4Exception as e:
+            if "Client '{}' doesn't exist".format(self.client_name) not in e.value:
+                self.structure.fail_current_block(e.value)
 
     def finalize(self):
         with Uninterruptible(self.out.log_exception) as run:

_universum/modules/vcs/swarm.py

@@ -20,13 +20,6 @@
 ]
 
 
-def check_request_result(result):
-    if result.status_code != 200:
-        text = "Invalid return code " + result.status_code + ". Response is:\n"
-        text += result.text
-        raise CiException(text)
-
-
 @needs_output
 class Swarm(ReportObserver, Module):
     """
@@ -154,7 +147,7 @@ def post_comment(self, text, filename=None, line=None, version=None, no_notifica
 
         result = requests.post(self.settings.server_url + "/api/v9/comments", data=request,
                                auth=(self.user, self.password))
-        check_request_result(result)
+        utils.check_request_result(result)
 
     def vote_review(self, result, version=None):
         request = {}
@@ -167,7 +160,7 @@ def vote_review(self, result, version=None):
 
         result = requests.patch(self.settings.server_url + "/api/v6/reviews/" + self.settings.review_id,
                                 data=request, auth=(self.user, self.password))
-        check_request_result(result)
+        utils.check_request_result(result)
 
     def report_start(self, report_text):
         self.update_review_version()

_universum/modules/vcs/vcs.py

@@ -5,7 +5,7 @@
 import shutil
 import sh
 
-from . import git_vcs, gerrit_vcs, perforce_vcs, local_vcs, base_vcs
+from . import git_vcs, github_vcs, gerrit_vcs, perforce_vcs, local_vcs, base_vcs
 from .. import artifact_collector
 from ..api_support import ApiSupport
 from ..project_directory import ProjectDirectory
@@ -27,25 +27,29 @@ def create_vcs(class_type=None):
         p4_driver_factory_class = perforce_vcs.PerforceSubmitVcs
         git_driver_factory_class = git_vcs.GitSubmitVcs
         gerrit_driver_factory_class = gerrit_vcs.GerritSubmitVcs
+        github_driver_factory_class = git_vcs.GitSubmitVcs
         local_driver_factory_class = base_vcs.BaseSubmitVcs
     elif class_type == "poll":
         p4_driver_factory_class = perforce_vcs.PerforcePollVcs
         git_driver_factory_class = git_vcs.GitPollVcs
         gerrit_driver_factory_class = git_vcs.GitPollVcs
+        github_driver_factory_class = git_vcs.GitPollVcs
         local_driver_factory_class = base_vcs.BasePollVcs
     else:
         p4_driver_factory_class = perforce_vcs.PerforceMainVcs
         git_driver_factory_class = git_vcs.GitMainVcs
         gerrit_driver_factory_class = gerrit_vcs.GerritMainVcs
+        github_driver_factory_class = github_vcs.GithubMainVcs
         local_driver_factory_class = local_vcs.LocalMainVcs
 
-    vcs_types = ["none", "p4", "git", "gerrit"]
+    vcs_types = ["none", "p4", "git", "gerrit", "github"]
 
     @needs_structure
     class Vcs(ProjectDirectory):
         local_driver_factory = Dependency(local_driver_factory_class)
         git_driver_factory = Dependency(git_driver_factory_class)
         gerrit_driver_factory = Dependency(gerrit_driver_factory_class)
+        github_driver_factory = Dependency(github_driver_factory_class)
         perforce_driver_factory = Dependency(p4_driver_factory_class)
 
         @staticmethod
@@ -55,7 +59,7 @@ def define_arguments(argument_parser):
             parser.add_argument("--vcs-type", "-vt", dest="type",
                                 choices=vcs_types, metavar="VCS_TYPE",
                                 help="Select repository type to download sources from: Perforce ('p4'), "
-                                     "Git ('git'), Gerrit ('gerrit') or a local directory ('none'). "
+                                     "Git ('git'), Gerrit ('gerrit'), GitHub ('github') or a local directory ('none'). "
                                      "Gerrit uses Git parameters. Each VCS type has its own settings.")
 
         def __init__(self, *args, **kwargs):
@@ -75,9 +79,9 @@ def __init__(self, *args, **kwargs):
                     Each of these types requires supplying its own
                     configuration parameters. At the minimum, the following
                     parameters are required:
-                      * "git" and "gerrit" - GIT_REPO (-gr) and GIT_REFSPEC (-grs)
-                      * "perforce"         - P4PORT (-p4p), P4USER (-p4u) and P4PASSWD (-p4P)
-                      * "none"             - SOURCE_DIR (-fsd)
+                      * "git", "github" and "gerrit" - GIT_REPO (-gr) and GIT_REFSPEC (-grs)
+                      * "perforce"                   - P4PORT (-p4p), P4USER (-p4u) and P4PASSWD (-p4P)
+                      * "none"                       - SOURCE_DIR (-fsd)
                       
                     Depending on the requested action, additional type-specific
                     parameters are required. For example, P4CLIENT (-p4c) is
@@ -91,6 +95,8 @@ def __init__(self, *args, **kwargs):
                     driver_factory = self.git_driver_factory
                 elif self.settings.type == "gerrit":
                     driver_factory = self.gerrit_driver_factory
+                elif self.settings.type == "github":
+                    driver_factory = self.github_driver_factory
                 else:
                     driver_factory = self.perforce_driver_factory
             except AttributeError: