move adviser requests from keb admin to investigator

thoth-station · Jul 7, 2022 · cd9097e · cd9097e
1 parent b78ea8f
commit cd9097e
Show file tree

Hide file tree

Showing 24 changed files with 1,659 additions and 400 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -20,6 +20,7 @@ repos:
       - id: check-yaml
       - id: debug-statements
       - id: detect-private-key
+        exclude: tests/keys/
       - id: end-of-file-fixer
       - id: name-tests-test
       - id: trailing-whitespace
@@ -28,6 +29,8 @@ repos:
     rev: 6.1.1
     hooks:
       - id: pydocstyle
+        exclude: ^tests/
+        additional_dependencies: ["toml"]
 
   - repo: https://github.com/psf/black
     rev: 22.3.0

diff --git a/Pipfile b/Pipfile
@@ -11,16 +11,23 @@ prometheus-async = "*"
 confluent-kafka = "*"
 thoth-common = "*"
 thoth-messaging = "*"
+aiohttp = "*"
+pyjwt = "*"
+thamos = "*"
+semver = "*"
+types-pyyaml = "*"
+cryptography = "*"
 
 [dev-packages]
 pytest = "*"
 pytest-timeout = "*"
 pytest-cov = "*"
 hypothesis-auto = "*"
-hypothesis = "*"
+hypothesis = "<6.47"
 pytest-mypy = "*"
 mypy = "*"
 pre-commit = "==2.0.1"
+pytest-aiohttp = "*"
 
 [requires]
 python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/consumer.py b/consumer.py
@@ -52,6 +52,7 @@
     missing_handler,
     current_consumer_offset,
 )
+from thoth.investigator.github_service import GithubService
 
 from thoth.common import OpenShift, init_logging
 from thoth.storages.graph import GraphDatabase
@@ -117,6 +118,9 @@
 halted_partitions = []  # type: List[TopicPartition]
 
 c = None  # type: Optional[Consumer]
+gh_service = GithubService(
+    app_id=os.getenv("KEBECHET_APP_ID"), app_private_key_path=os.getenv("KEBECHET_APP_PRIVATE_KEY_PATH")
+)
 
 
 def _handler_lookup(topic_name, version, table=investigator_handler_table, default=None):
@@ -209,7 +213,13 @@ async def _worker(q: asyncio.Queue):
         contents = json.loads(msg.value().decode("utf-8"))
         for i in range(0, Configuration.MAX_RETRIES):
             try:
-                await func(contents, openshift=openshift, graph=graph, msg=msg)
+                await func(
+                    contents,
+                    openshift=openshift,
+                    graph=graph,
+                    gh_service=gh_service,
+                    msg=msg,
+                )
                 c.commit(message=msg)
                 current_consumer_offset.labels(topic_name=msg.topic(), partition=msg.partition()).set(msg.offset())
                 break

diff --git a/tests/github_api_responses/get_installation_token_success.json b/tests/github_api_responses/get_installation_token_success.json
@@ -0,0 +1,134 @@
+{
+    "token": "ghs_16C7e42F292c6912E7710c838347Ae178B4a",
+    "expires_at": "2016-07-11T22:14:10Z",
+    "permissions": {
+        "issues": "write",
+        "contents": "read"
+    },
+    "repository_selection": "selected",
+    "repositories": [
+        {
+            "id": 1296269,
+            "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5",
+            "name": "Hello-World",
+            "full_name": "octocat/Hello-World",
+            "owner": {
+                "login": "octocat",
+                "id": 1,
+                "node_id": "MDQ6VXNlcjE=",
+                "avatar_url": "https://github.com/images/error/octocat_happy.gif",
+                "gravatar_id": "",
+                "url": "https://api.github.com/users/octocat",
+                "html_url": "https://github.com/octocat",
+                "followers_url": "https://api.github.com/users/octocat/followers",
+                "following_url": "https://api.github.com/users/octocat/following{/other_user}",
+                "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}",
+                "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}",
+                "subscriptions_url": "https://api.github.com/users/octocat/subscriptions",
+                "organizations_url": "https://api.github.com/users/octocat/orgs",
+                "repos_url": "https://api.github.com/users/octocat/repos",
+                "events_url": "https://api.github.com/users/octocat/events{/privacy}",
+                "received_events_url": "https://api.github.com/users/octocat/received_events",
+                "type": "User",
+                "site_admin": false
+            },
+            "private": false,
+            "html_url": "https://github.com/octocat/Hello-World",
+            "description": "This your first repo!",
+            "fork": false,
+            "url": "https://api.github.com/repos/octocat/Hello-World",
+            "archive_url": "https://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}",
+            "assignees_url": "https://api.github.com/repos/octocat/Hello-World/assignees{/user}",
+            "blobs_url": "https://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}",
+            "branches_url": "https://api.github.com/repos/octocat/Hello-World/branches{/branch}",
+            "collaborators_url": "https://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}",
+            "comments_url": "https://api.github.com/repos/octocat/Hello-World/comments{/number}",
+            "commits_url": "https://api.github.com/repos/octocat/Hello-World/commits{/sha}",
+            "compare_url": "https://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}",
+            "contents_url": "https://api.github.com/repos/octocat/Hello-World/contents/{+path}",
+            "contributors_url": "https://api.github.com/repos/octocat/Hello-World/contributors",
+            "deployments_url": "https://api.github.com/repos/octocat/Hello-World/deployments",
+            "downloads_url": "https://api.github.com/repos/octocat/Hello-World/downloads",
+            "events_url": "https://api.github.com/repos/octocat/Hello-World/events",
+            "forks_url": "https://api.github.com/repos/octocat/Hello-World/forks",
+            "git_commits_url": "https://api.github.com/repos/octocat/Hello-World/git/commits{/sha}",
+            "git_refs_url": "https://api.github.com/repos/octocat/Hello-World/git/refs{/sha}",
+            "git_tags_url": "https://api.github.com/repos/octocat/Hello-World/git/tags{/sha}",
+            "git_url": "git:github.com/octocat/Hello-World.git",
+            "issue_comment_url": "https://api.github.com/repos/octocat/Hello-World/issues/comments{/number}",
+            "issue_events_url": "https://api.github.com/repos/octocat/Hello-World/issues/events{/number}",
+            "issues_url": "https://api.github.com/repos/octocat/Hello-World/issues{/number}",
+            "keys_url": "https://api.github.com/repos/octocat/Hello-World/keys{/key_id}",
+            "labels_url": "https://api.github.com/repos/octocat/Hello-World/labels{/name}",
+            "languages_url": "https://api.github.com/repos/octocat/Hello-World/languages",
+            "merges_url": "https://api.github.com/repos/octocat/Hello-World/merges",
+            "milestones_url": "https://api.github.com/repos/octocat/Hello-World/milestones{/number}",
+            "notifications_url": "https://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}",
+            "pulls_url": "https://api.github.com/repos/octocat/Hello-World/pulls{/number}",
+            "releases_url": "https://api.github.com/repos/octocat/Hello-World/releases{/id}",
+            "ssh_url": "[email protected]:octocat/Hello-World.git",
+            "stargazers_url": "https://api.github.com/repos/octocat/Hello-World/stargazers",
+            "statuses_url": "https://api.github.com/repos/octocat/Hello-World/statuses/{sha}",
+            "subscribers_url": "https://api.github.com/repos/octocat/Hello-World/subscribers",
+            "subscription_url": "https://api.github.com/repos/octocat/Hello-World/subscription",
+            "tags_url": "https://api.github.com/repos/octocat/Hello-World/tags",
+            "teams_url": "https://api.github.com/repos/octocat/Hello-World/teams",
+            "trees_url": "https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}",
+            "clone_url": "https://github.com/octocat/Hello-World.git",
+            "mirror_url": "git:git.example.com/octocat/Hello-World",
+            "hooks_url": "https://api.github.com/repos/octocat/Hello-World/hooks",
+            "svn_url": "https://svn.github.com/octocat/Hello-World",
+            "homepage": "https://github.com",
+            "language": null,
+            "forks_count": 9,
+            "stargazers_count": 80,
+            "watchers_count": 80,
+            "size": 108,
+            "default_branch": "master",
+            "open_issues_count": 0,
+            "is_template": true,
+            "topics": [
+                "octocat",
+                "atom",
+                "electron",
+                "api"
+            ],
+            "has_issues": true,
+            "has_projects": true,
+            "has_wiki": true,
+            "has_pages": false,
+            "has_downloads": true,
+            "archived": false,
+            "disabled": false,
+            "visibility": "public",
+            "pushed_at": "2011-01-26T19:06:43Z",
+            "created_at": "2011-01-26T19:01:12Z",
+            "updated_at": "2011-01-26T19:14:43Z",
+            "permissions": {
+                "admin": false,
+                "push": false,
+                "pull": true
+            },
+            "allow_rebase_merge": true,
+            "template_repository": null,
+            "temp_clone_token": "ABTLWHOULUVAXGTRYU7OC2876QJ2O",
+            "allow_squash_merge": true,
+            "allow_auto_merge": false,
+            "delete_branch_on_merge": true,
+            "allow_merge_commit": true,
+            "subscribers_count": 42,
+            "network_count": 0,
+            "license": {
+                "key": "mit",
+                "name": "MIT License",
+                "url": "https://api.github.com/licenses/mit",
+                "spdx_id": "MIT",
+                "node_id": "MDc6TGljZW5zZW1pdA==",
+                "html_url": "https://github.com/licenses/mit"
+            },
+            "forks": 1,
+            "open_issues": 1,
+            "watchers": 1
+        }
+    ]
+}
diff --git a/tests/github_api_responses/get_repo_installation_success.json b/tests/github_api_responses/get_repo_installation_success.json
@@ -0,0 +1,50 @@
+{
+    "id": 1,
+    "account": {
+        "login": "github",
+        "id": 1,
+        "node_id": "MDEyOk9yZ2FuaXphdGlvbjE=",
+        "avatar_url": "https://github.com/images/error/hubot_happy.gif",
+        "gravatar_id": "",
+        "url": "https://api.github.com/orgs/github",
+        "html_url": "https://github.com/github",
+        "followers_url": "https://api.github.com/users/github/followers",
+        "following_url": "https://api.github.com/users/github/following{/other_user}",
+        "gists_url": "https://api.github.com/users/github/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/github/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/github/subscriptions",
+        "organizations_url": "https://api.github.com/users/github/orgs",
+        "repos_url": "https://api.github.com/orgs/github/repos",
+        "events_url": "https://api.github.com/orgs/github/events",
+        "received_events_url": "https://api.github.com/users/github/received_events",
+        "type": "Organization",
+        "site_admin": false
+    },
+    "repository_selection": "all",
+    "access_tokens_url": "https://api.github.com/installations/1/access_tokens",
+    "repositories_url": "https://api.github.com/installation/repositories",
+    "html_url": "https://github.com/organizations/github/settings/installations/1",
+    "app_id": 1,
+    "target_id": 1,
+    "target_type": "Organization",
+    "permissions": {
+        "checks": "write",
+        "metadata": "read",
+        "contents": "read"
+    },
+    "events": [
+        "push",
+        "pull_request"
+    ],
+    "created_at": "2018-02-09T20:51:14Z",
+    "updated_at": "2018-02-09T20:51:14Z",
+    "single_file_name": "config.yml",
+    "has_multiple_single_files": true,
+    "single_file_paths": [
+        "config.yml",
+        ".github/issue_TEMPLATE.md"
+    ],
+    "app_slug": "github-actions",
+    "suspended_at": null,
+    "suspended_by": null
+}
diff --git a/tests/github_service_test.py b/tests/github_service_test.py
@@ -0,0 +1,63 @@
+"""Testing for async github service classes."""
+
+import json
+import os
+from unittest.mock import MagicMock
+import jwt
+import pytest
+from thoth.investigator.github_service import GithubApp
+
+PUBLICKEY_FILENAME = os.path.join(os.path.dirname(__file__), "keys/jwt-key.pub")
+PRIVATEKEY_FILENAME = os.path.join(os.path.dirname(__file__), "keys/jwt-key")
+
+GITHUBAPI_RESPONSES_DIR = os.path.join(os.path.dirname(__file__), "github_api_responses")
+
+
+def _decrypt_jwt(jw_token: str, publickey):
+    return jwt.decode(jwt=jw_token.encode(), key=publickey, algorithms=["RS256"])
+
+
+async def get_json_file_content(file_name) -> dict:
+    with open(file_name, "r") as f:
+        return json.load(f)
+
+
+class TestGithubApp:
+    def setup(self):
+        with open(PUBLICKEY_FILENAME, "r") as f:
+            self.public_key = f.read()
+        self.github_app = GithubApp("kebechet", PRIVATEKEY_FILENAME)
+
+    def test_generate_jwt(self):
+        jw_token = self.github_app._generate_jwt()
+        content: dict = _decrypt_jwt(jw_token, self.public_key)
+        assert content["iss"] == "kebechet"
+        assert "iat" in content
+        assert "exp" in content
+
+    @pytest.mark.asyncio
+    async def test_get_repo_installation_info(self):
+        self.setup()
+        mock = self.github_app._session
+        mock.get = MagicMock()
+        mock.get.return_value.__aenter__.return_value.json = lambda: get_json_file_content(
+            os.path.join(GITHUBAPI_RESPONSES_DIR, "get_repo_installation_success.json")
+        )
+        mock.get.return_value.__aenter__.return_value.status = 200
+        id = await self.github_app._get_installation_id_for_repo("github", "octocat")
+        assert mock.get.call_count == 1
+        assert id == 1
+
+    @pytest.mark.asyncio
+    async def test_get_repo_installation_token(self):
+        self.setup()
+        self.github_app._get_installation_id_for_repo = MagicMock()
+        self.github_app._get_installation_id_for_repo.return_value = 1
+        mock = self.github_app._session
+        mock.post = MagicMock()
+        mock.post.return_value.__aenter__.return_value.json = lambda: get_json_file_content(
+            os.path.join(GITHUBAPI_RESPONSES_DIR, "get_installation_token_success.json")
+        )
+        mock.post.return_value.__aenter__.return_value.status = 201
+        token = await self.github_app.get_repository_token("foo", "bar")
+        assert token.token == "ghs_16C7e42F292c6912E7710c838347Ae178B4a"
diff --git a/tests/investigate_unresolved_package_test.py b/tests/investigate_unresolved_package_test.py
@@ -18,7 +18,7 @@
 """Test investigator."""
 
 from tests.base_test import InvestigatorTestCase
-from thoth.investigator.investigate_unresolved_package import investigate_unresolved_package
+from thoth.investigator.unresolved_package import investigate_unresolved_package
 
 
 class TestProducer(InvestigatorTestCase):

diff --git a/tests/keys/jwt-key b/tests/keys/jwt-key
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQCqKu7vdySxrNV0
+CxiUEPwQlp8Y5rhL1iNndQhU8KgQsXv601t7nfEBV0OIr9Brqli0QjEXWSZACBOx
+dbM+tU5fdHKMx8eICuxNBKDE7OojOdCMmF4IfMeGGnpO7IuSn3pv0FaKGb232prd
+VqW5XJ7eO5BMido0kn3KhqmWunkosw78390phPXxzHpyOpngTlP4olAj7EoJqo6K
+lFfeYm7hB0ob+pxoLWYsKMafS1lexWFoSOT6dtT37As2sof/RjeWtZYd05M8ImhX
+LntCeBY9aEGkfXn/3y6U6NrMAvXgtHBNb6zjSJ+zXaytHdycmHiKlINX8tvrETYU
+BUfQ4LO8EwYe57a+bsAbX0XFslw8GGQROpPUwe1JV+yGZkhVnLWcEtptzWk3iqqE
+pzsAKyP1mtMLdlF5rTP6sFRRInz4rth3o458oz9Ttj2LXURTUTy26PDGu6axfMMh
+Q9hBII6pQUnlArWRXHjsawH+hJJ8255mBTA60DoF4mK49/oeecFRyB+fEGM3bBFO
+7rU0+Rb3eU1WioaKTpOWPRClaqp/bMKMuEaXcKrXwJMbvoo848YgLJNO8UeAcecy
+SZrI1V9a/R+GqO/0IToVkw009o9ZINtL/VLnVohJvhWRGfIOCzGm68abgj+4Nkm9
+yftqx/PoVNEHutzRL35ZI1YSdX7bYQIDAQABAoICAAHGSYp5TUC8S8CG4WYlKgwD
+BtPNyZmmUFmwWQI+e1LtPe5Gf2gNDSIZZquyQU73GBBavtdvDTAzXKKLzv6VKMAt
+ggx8bUlN+wIM2TL1/ALoTaLtWEUWilYvob5gr5P9czDjKktiDqP9wIoU4Tl13Ebv
+GuaBrPhsrxXldXVheBxhjxp4BamEcGaE3lQGO6odBoNNhO5W2dO3OKHlHnfTi/WG
+pQKscS6KFyWSaUlsX3J5m2h8QcGWuPVCrF8z+JnbKJYf5XjaL3yY8njaCGfpD5AC
+d2XekoHmkxdvcz5dnXoqKAjeNftrw3JYNPbUajW+8R8bobVgvP6WF2ChAr26+PD3
+fBIB04TNwP+D7LZK6AmXgoe7ohaiQtcHVz56TqikrPkIHnX2RJR8fv/+WjCAM5G7
+zsNXBjtghl+80Adw8YUUkXxTcTTsAX8irnl6tkF8c4ZgkFy2yYuPG/dYoh2o4zgq
+jXJVLKhkWG64cGwnpYdl3cxZAynW0k7jtmoAqRFQHa2G0hmL65ftXOjwEAlB+Bdf
+D99vQjrDuFxBlnJBosdZEZpbjLrO7zzazc0AOQOyUn2RI7aHJEOurUxjihH61MZq
+2oADJGl+h6t8rQe5IbvP5kN+Fuw0B6Dcu5BIj5I6eSWAumgcyc3EKlzjCnOO5FVB
+wPe7SYxwFtApWwdKhb3/AoIBAQDW9DDt17uJ7s3zSAia2msRB0yvlkh/osfhhf2h
+drHyj/rAPu/xfgInMt1rjj/caE26SY9L/vq53Z9e9PqOgikbQWDTycqgLD6pZCwD
+otO5c90k094yzGJLwrKjFUSLTAak6PuKoJJnLexc51vvmh958YFbH9qX0m2LkA6W
+Hu004cCfuyc5+YU1GKUplHvi/O413nFuLL07njjlLP2GBesnBndcksM7K3R+aAO2
+EAMDmivjlaPzoxVQNz4SqEOnVCfXl12vxgf+1l7d95PBZPGDoDn5+oQ3y094wDc1
+OOj1GSelfCAXHU0EZ4f5nrDrYZ+UWzpmME58uQqSZOoNq6IrAoIBAQDKqWokLs9L
+DIVTNOg/oK+GtODKZQ1cyEE0wkhUBTgDas27eXXD8nxUsZ0/qJKH7RWbGY/hvyvV
+rKwFKl8ELJYkf32euH+utM5wNO4aAEV6kEHnl0jwBnEoVlcZqt1D0u3Q2FTKc9uj
+lPuQFz7m3BWqsIqwgFmqS1xDVVoKtm56PhO4hGKao2SAOOR5qRiiorg/GisULd1k
+gzIme58wuyl3sN44qII0w+gAbzcI3+n56c8uiQEXAnaYXIG3tAy3giu4bj7SQ7p2
+WLhDcIwwEu9wXeO+2L10wsL1nEyD7VRhN3ckrlxS4X0ii7bxSIeqKG2B1sEvtoF1
+6QfJnn3qKM6jAoIBAHAfdfpKYpejSihRdbOYpOp//T9Fy9FQDqUTqGX1rQKSmcB3
+098K1IKv5JF65NLKUWkPf8CrYwtakAwPCzcUq+58ns46XwhHq/PdvZuz7m65Aix4
+m1myBL/x8g5hRJzIblrM8JCyC0vox3b6c5k6spqAH/JxXit9Zwi6fvFNdnv0Ge73
+x5FVF2fH0DctLaYMOxE8U7ydd6PGd3ULj76+DZgQpmhSYaXWl/BrvyvTlE7nt2cC
+IkAu3Co4dpdyz79ThWjOmt7aFEeYeGmd/6OMTPJUvIkpZJhZ471fvixz4AghM+39
+HhwQe6EFhAbwAOb2yC+xmne9UON7/gi4vOGemnsCggEACPL+OubORJQi9w+pCdZA
+qgQmwQlBU82MA5Zotz0bwhrfRt4QMzzr/JlzRjJddh8kCxtP3Toh52eemQPL+lU1
+tBrlIzMmgEJ/szjDIubMKqNgJafqkp7WNF/LrNh8uNgBpG1JIT+IfWrAyRYdsvgZ
+BNaS6URhsUFuq5KDTWHM8xz2pflCmW2JrJfWjwDGk7f3xuGS8WstEZkxh72BxBcf
+t6dZyMq4JThZcFt45K1umZiXi4iFQczcC9JZ8BDFVv0qjRRhNcvD2iIE0QWRSzYq
+2J1sxyqvAw7uqQXj4xeB2GEoo7gHX6mRawscYv8+7mamn0sqcL7wbJKDw4V6LfLr
+XQKCAQB7Hp61RNPN2WmXH1ctVIrLnj/QmDHPZygASpdjBnRcbxG319bgpMNWDJLd
+rrSHJOX66AlobzisF6liCYhdLMorAXffzX19cS2iB8Ft3T7QJQ5lpV4eHb5rM5vt
+Fz6MWj7chzUrCXEw8NjiMS0aL5ADf9qAH4i6gfc/rAvvJl5vSCXJwu4Nm6Jgw0hW
+LP/viRt1jW3J/nE86iMamlvCxGX+A8HzqvlXixdGeVHFYQqt61kfSqnSojezRSPV
+uayF3/shCRbRfg6rk1FJFChvCecV8NaiIuTKugXoViKXmUAvPQp2mcyVr2ar6mH2
+Nqp0u3nM2LBCb1WPY31MGdcSX/qp
+-----END PRIVATE KEY-----