diff --git a/Pipfile b/Pipfile index 239b3fc..87b7caf 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,8 @@ prometheus-async = "*" confluent-kafka = "*" thoth-common = "*" thoth-messaging = "*" +aiohttp = "*" +pyjwt = "*" [dev-packages] pytest = "*" diff --git a/Pipfile.lock b/Pipfile.lock index c1249e1..7fef899 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c24b700eb6944c397e437f7a9797d67a5041093abd3ee8503fc7aacaaa70cc39" + "sha256": "a7e887e61e4ebecb096f3aba13ddd90884452822c7d1649ff6d9daae91f9fc9b" }, "pipfile-spec": 6, "requires": { @@ -91,7 +91,7 @@ "sha256:fa0ffcace9b3aa34d205d8130f7873fcfefcb6a4dd3dd705b0dab69af6712642", "sha256:fc5471e1a54de15ef71c1bc6ebe80d4dc681ea600e68bfd1cbce40427f0b7578" ], - "markers": "python_version >= '3.6'", + "index": "pypi", "version": "==3.8.1" }, "aiosignal": { @@ -179,19 +179,19 @@ }, "boto3": { "hashes": [ - "sha256:1e72294b042651ab27a3b6a9eea2810fc8faab5be52b0876f9f5b55f5fd9e101", - "sha256:4110e8a1790842d1f818363b83c5f72dea4bdd980c46df1540814c4a79d72884" + "sha256:c9f37d93b56f24bd5dac608d7541799a596da2fc85119af65815fe30c7a36a4c", + "sha256:fa30deb141f12cd51b226638f831ff1be5ce60704063a4634bbca554d2fed9ea" ], "markers": "python_version >= '3.6'", - "version": "==1.20.33" + "version": "==1.20.39" }, "botocore": { "hashes": [ - "sha256:8faeda0da14a3cb5df8005052527cc0263181adad6bdbfedbe20079c72973c2c", - "sha256:9be1ea775e6de420a5873dff42108a1843e9137a52ef98c926dad56a8152ef2f" + "sha256:3a373890c953f9f5dc3d65aeea4efcab9e1e1a7ef5fb9abfb247d8a8ecf0f8ca", + "sha256:9ea02d96aa425427cee286d9e9f1fb9cb81db1c3a655c08814d952cd9ed75627" ], "markers": "python_version >= '3.6'", - "version": "==1.23.33" + "version": "==1.23.39" }, "cachetools": { "hashes": [ @@ -332,81 +332,68 @@ }, "frozenlist": { "hashes": [ - "sha256:01d79515ed5aa3d699b05f6bdcf1fe9087d61d6b53882aa599a10853f0479c6c", - "sha256:0a7c7cce70e41bc13d7d50f0e5dd175f14a4f1837a8549b0936ed0cbe6170bf9", - "sha256:11ff401951b5ac8c0701a804f503d72c048173208490c54ebb8d7bb7c07a6d00", - "sha256:14a5cef795ae3e28fb504b73e797c1800e9249f950e1c964bb6bdc8d77871161", - "sha256:16eef427c51cb1203a7c0ab59d1b8abccaba9a4f58c4bfca6ed278fc896dc193", - "sha256:16ef7dd5b7d17495404a2e7a49bac1bc13d6d20c16d11f4133c757dd94c4144c", - "sha256:181754275d5d32487431a0a29add4f897968b7157204bc1eaaf0a0ce80c5ba7d", - "sha256:1cf63243bc5f5c19762943b0aa9e0d3fb3723d0c514d820a18a9b9a5ef864315", - "sha256:1cfe6fef507f8bac40f009c85c7eddfed88c1c0d38c75e72fe10476cef94e10f", - "sha256:1fef737fd1388f9b93bba8808c5f63058113c10f4e3c0763ced68431773f72f9", - "sha256:25b358aaa7dba5891b05968dd539f5856d69f522b6de0bf34e61f133e077c1a4", - "sha256:26f602e380a5132880fa245c92030abb0fc6ff34e0c5500600366cedc6adb06a", - "sha256:28e164722ea0df0cf6d48c4d5bdf3d19e87aaa6dfb39b0ba91153f224b912020", - "sha256:2de5b931701257d50771a032bba4e448ff958076380b049fd36ed8738fdb375b", - "sha256:3457f8cf86deb6ce1ba67e120f1b0128fcba1332a180722756597253c465fc1d", - "sha256:351686ca020d1bcd238596b1fa5c8efcbc21bffda9d0efe237aaa60348421e2a", - "sha256:406aeb340613b4b559db78d86864485f68919b7141dec82aba24d1477fd2976f", - "sha256:41de4db9b9501679cf7cddc16d07ac0f10ef7eb58c525a1c8cbff43022bddca4", - "sha256:41f62468af1bd4e4b42b5508a3fe8cc46a693f0cdd0ca2f443f51f207893d837", - "sha256:4766632cd8a68e4f10f156a12c9acd7b1609941525569dd3636d859d79279ed3", - "sha256:47b2848e464883d0bbdcd9493c67443e5e695a84694efff0476f9059b4cb6257", - "sha256:4a495c3d513573b0b3f935bfa887a85d9ae09f0627cf47cad17d0cc9b9ba5c38", - "sha256:4ad065b2ebd09f32511ff2be35c5dfafee6192978b5a1e9d279a5c6e121e3b03", - "sha256:4c457220468d734e3077580a3642b7f682f5fd9507f17ddf1029452450912cdc", - "sha256:4f52d0732e56906f8ddea4bd856192984650282424049c956857fed43697ea43", - "sha256:54a1e09ab7a69f843cd28fefd2bcaf23edb9e3a8d7680032c8968b8ac934587d", - "sha256:5a72eecf37eface331636951249d878750db84034927c997d47f7f78a573b72b", - "sha256:5df31bb2b974f379d230a25943d9bf0d3bc666b4b0807394b131a28fca2b0e5f", - "sha256:66a518731a21a55b7d3e087b430f1956a36793acc15912e2878431c7aec54210", - "sha256:6790b8d96bbb74b7a6f4594b6f131bd23056c25f2aa5d816bd177d95245a30e3", - "sha256:68201be60ac56aff972dc18085800b6ee07973c49103a8aba669dee3d71079de", - "sha256:6e105013fa84623c057a4381dc8ea0361f4d682c11f3816cc80f49a1f3bc17c6", - "sha256:705c184b77565955a99dc360f359e8249580c6b7eaa4dc0227caa861ef46b27a", - "sha256:72cfbeab7a920ea9e74b19aa0afe3b4ad9c89471e3badc985d08756efa9b813b", - "sha256:735f386ec522e384f511614c01d2ef9cf799f051353876b4c6fb93ef67a6d1ee", - "sha256:82d22f6e6f2916e837c91c860140ef9947e31194c82aaeda843d6551cec92f19", - "sha256:83334e84a290a158c0c4cc4d22e8c7cfe0bba5b76d37f1c2509dabd22acafe15", - "sha256:84e97f59211b5b9083a2e7a45abf91cfb441369e8bb6d1f5287382c1c526def3", - "sha256:87521e32e18a2223311afc2492ef2d99946337da0779ddcda77b82ee7319df59", - "sha256:878ebe074839d649a1cdb03a61077d05760624f36d196884a5cafb12290e187b", - "sha256:89fdfc84c6bf0bff2ff3170bb34ecba8a6911b260d318d377171429c4be18c73", - "sha256:8b4c7665a17c3a5430edb663e4ad4e1ad457614d1b2f2b7f87052e2ef4fa45ca", - "sha256:8b54cdd2fda15467b9b0bfa78cee2ddf6dbb4585ef23a16e14926f4b076dfae4", - "sha256:94728f97ddf603d23c8c3dd5cae2644fa12d33116e69f49b1644a71bb77b89ae", - "sha256:954b154a4533ef28bd3e83ffdf4eadf39deeda9e38fb8feaf066d6069885e034", - "sha256:977a1438d0e0d96573fd679d291a1542097ea9f4918a8b6494b06610dfeefbf9", - "sha256:9ade70aea559ca98f4b1b1e5650c45678052e76a8ab2f76d90f2ac64180215a2", - "sha256:9b6e21e5770df2dea06cb7b6323fbc008b13c4a4e3b52cb54685276479ee7676", - "sha256:a0d3ffa8772464441b52489b985d46001e2853a3b082c655ec5fad9fb6a3d618", - "sha256:a37594ad6356e50073fe4f60aa4187b97d15329f2138124d252a5a19c8553ea4", - "sha256:a8d86547a5e98d9edd47c432f7a14b0c5592624b496ae9880fb6332f34af1edc", - "sha256:aa44c4740b4e23fcfa259e9dd52315d2b1770064cde9507457e4c4a65a04c397", - "sha256:acc4614e8d1feb9f46dd829a8e771b8f5c4b1051365d02efb27a3229048ade8a", - "sha256:af2a51c8a381d76eabb76f228f565ed4c3701441ecec101dd18be70ebd483cfd", - "sha256:b2ae2f5e9fa10805fb1c9adbfefaaecedd9e31849434be462c3960a0139ed729", - "sha256:b46f997d5ed6d222a863b02cdc9c299101ee27974d9bbb2fd1b3c8441311c408", - "sha256:bc93f5f62df3bdc1f677066327fc81f92b83644852a31c6aa9b32c2dde86ea7d", - "sha256:bfbaa08cf1452acad9cb1c1d7b89394a41e712f88df522cea1a0f296b57782a0", - "sha256:c1e8e9033d34c2c9e186e58279879d78c94dd365068a3607af33f2bc99357a53", - "sha256:c5328ed53fdb0a73c8a50105306a3bc013e5ca36cca714ec4f7bd31d38d8a97f", - "sha256:c6a9d84ee6427b65a81fc24e6ef589cb794009f5ca4150151251c062773e7ed2", - "sha256:c98d3c04701773ad60d9545cd96df94d955329efc7743fdb96422c4b669c633b", - "sha256:cb3957c39668d10e2b486acc85f94153520a23263b6401e8f59422ef65b9520d", - "sha256:e63ad0beef6ece06475d29f47d1f2f29727805376e09850ebf64f90777962792", - "sha256:e74f8b4d8677ebb4015ac01fcaf05f34e8a1f22775db1f304f497f2f88fdc697", - "sha256:e7d0dd3e727c70c2680f5f09a0775525229809f1a35d8552b92ff10b2b14f2c2", - "sha256:ec6cf345771cdb00791d271af9a0a6fbfc2b6dd44cb753f1eeaa256e21622adb", - "sha256:ed58803563a8c87cf4c0771366cf0ad1aa265b6b0ae54cbbb53013480c7ad74d", - "sha256:f0081a623c886197ff8de9e635528fd7e6a387dccef432149e25c13946cb0cd0", - "sha256:f025f1d6825725b09c0038775acab9ae94264453a696cc797ce20c0769a7b367", - "sha256:f5f3b2942c3b8b9bfe76b408bbaba3d3bb305ee3693e8b1d631fe0a0d4f93673", - "sha256:fbd4844ff111449f3bbe20ba24fbb906b5b1c2384d0f3287c9f7da2354ce6d23" + "sha256:006d3595e7d4108a12025ddf415ae0f6c9e736e726a5db0183326fd191b14c5e", + "sha256:01a73627448b1f2145bddb6e6c2259988bb8aee0fb361776ff8604b99616cd08", + "sha256:03a7dd1bfce30216a3f51a84e6dd0e4a573d23ca50f0346634916ff105ba6e6b", + "sha256:0437fe763fb5d4adad1756050cbf855bbb2bf0d9385c7bb13d7a10b0dd550486", + "sha256:04cb491c4b1c051734d41ea2552fde292f5f3a9c911363f74f39c23659c4af78", + "sha256:0c36e78b9509e97042ef869c0e1e6ef6429e55817c12d78245eb915e1cca7468", + "sha256:25af28b560e0c76fa41f550eacb389905633e7ac02d6eb3c09017fa1c8cdfde1", + "sha256:2fdc3cd845e5a1f71a0c3518528bfdbfe2efaf9886d6f49eacc5ee4fd9a10953", + "sha256:30530930410855c451bea83f7b272fb1c495ed9d5cc72895ac29e91279401db3", + "sha256:31977f84828b5bb856ca1eb07bf7e3a34f33a5cddce981d880240ba06639b94d", + "sha256:3c62964192a1c0c30b49f403495911298810bada64e4f03249ca35a33ca0417a", + "sha256:3f7c935c7b58b0d78c0beea0c7358e165f95f1fd8a7e98baa40d22a05b4a8141", + "sha256:40dff8962b8eba91fd3848d857203f0bd704b5f1fa2b3fc9af64901a190bba08", + "sha256:40ec383bc194accba825fbb7d0ef3dda5736ceab2375462f1d8672d9f6b68d07", + "sha256:436496321dad302b8b27ca955364a439ed1f0999311c393dccb243e451ff66aa", + "sha256:4406cfabef8f07b3b3af0f50f70938ec06d9f0fc26cbdeaab431cbc3ca3caeaa", + "sha256:45334234ec30fc4ea677f43171b18a27505bfb2dba9aca4398a62692c0ea8868", + "sha256:47be22dc27ed933d55ee55845d34a3e4e9f6fee93039e7f8ebadb0c2f60d403f", + "sha256:4a44ebbf601d7bac77976d429e9bdb5a4614f9f4027777f9e54fd765196e9d3b", + "sha256:4eda49bea3602812518765810af732229b4291d2695ed24a0a20e098c45a707b", + "sha256:57f4d3f03a18facacb2a6bcd21bccd011e3b75d463dc49f838fd699d074fabd1", + "sha256:603b9091bd70fae7be28bdb8aa5c9990f4241aa33abb673390a7f7329296695f", + "sha256:65bc6e2fece04e2145ab6e3c47428d1bbc05aede61ae365b2c1bddd94906e478", + "sha256:691ddf6dc50480ce49f68441f1d16a4c3325887453837036e0fb94736eae1e58", + "sha256:6983a31698490825171be44ffbafeaa930ddf590d3f051e397143a5045513b01", + "sha256:6a202458d1298ced3768f5a7d44301e7c86defac162ace0ab7434c2e961166e8", + "sha256:6eb275c6385dd72594758cbe96c07cdb9bd6becf84235f4a594bdf21e3596c9d", + "sha256:754728d65f1acc61e0f4df784456106e35afb7bf39cfe37227ab00436fb38676", + "sha256:768efd082074bb203c934e83a61654ed4931ef02412c2fbdecea0cff7ecd0274", + "sha256:772965f773757a6026dea111a15e6e2678fbd6216180f82a48a40b27de1ee2ab", + "sha256:871d42623ae15eb0b0e9df65baeee6976b2e161d0ba93155411d58ff27483ad8", + "sha256:88aafd445a233dbbf8a65a62bc3249a0acd0d81ab18f6feb461cc5a938610d24", + "sha256:8c905a5186d77111f02144fab5b849ab524f1e876a1e75205cd1386a9be4b00a", + "sha256:8cf829bd2e2956066dd4de43fd8ec881d87842a06708c035b37ef632930505a2", + "sha256:92e650bd09b5dda929523b9f8e7f99b24deac61240ecc1a32aeba487afcd970f", + "sha256:93641a51f89473837333b2f8100f3f89795295b858cd4c7d4a1f18e299dc0a4f", + "sha256:94c7a8a9fc9383b52c410a2ec952521906d355d18fccc927fca52ab575ee8b93", + "sha256:9f892d6a94ec5c7b785e548e42722e6f3a52f5f32a8461e82ac3e67a3bd073f1", + "sha256:acb267b09a509c1df5a4ca04140da96016f40d2ed183cdc356d237286c971b51", + "sha256:adac9700675cf99e3615eb6a0eb5e9f5a4143c7d42c05cea2e7f71c27a3d0846", + "sha256:aff388be97ef2677ae185e72dc500d19ecaf31b698986800d3fc4f399a5e30a5", + "sha256:b5009062d78a8c6890d50b4e53b0ddda31841b3935c1937e2ed8c1bda1c7fb9d", + "sha256:b684c68077b84522b5c7eafc1dc735bfa5b341fb011d5552ebe0968e22ed641c", + "sha256:b9e3e9e365991f8cc5f5edc1fd65b58b41d0514a6a7ad95ef5c7f34eb49b3d3e", + "sha256:bd89acd1b8bb4f31b47072615d72e7f53a948d302b7c1d1455e42622de180eae", + "sha256:bde99812f237f79eaf3f04ebffd74f6718bbd216101b35ac7955c2d47c17da02", + "sha256:c6c321dd013e8fc20735b92cb4892c115f5cdb82c817b1e5b07f6b95d952b2f0", + "sha256:ce6f2ba0edb7b0c1d8976565298ad2deba6f8064d2bebb6ffce2ca896eb35b0b", + "sha256:d2257aaba9660f78c7b1d8fea963b68f3feffb1a9d5d05a18401ca9eb3e8d0a3", + "sha256:d26b650b71fdc88065b7a21f8ace70175bcf3b5bdba5ea22df4bfd893e795a3b", + "sha256:d6d32ff213aef0fd0bcf803bffe15cfa2d4fde237d1d4838e62aec242a8362fa", + "sha256:e1e26ac0a253a2907d654a37e390904426d5ae5483150ce3adedb35c8c06614a", + "sha256:e30b2f9683812eb30cf3f0a8e9f79f8d590a7999f731cf39f9105a7c4a39489d", + "sha256:e84cb61b0ac40a0c3e0e8b79c575161c5300d1d89e13c0e02f76193982f066ed", + "sha256:e982878792c971cbd60ee510c4ee5bf089a8246226dea1f2138aa0bb67aff148", + "sha256:f20baa05eaa2bcd5404c445ec51aed1c268d62600362dc6cfe04fae34a424bd9", + "sha256:f7353ba3367473d1d616ee727945f439e027f0bb16ac1a750219a8344d1d5d3c", + "sha256:f96293d6f982c58ebebb428c50163d010c2f05de0cde99fd681bfdc18d4b2dc2", + "sha256:ff9310f05b9d9c5c4dd472983dc956901ee6cb2c3ec1ab116ecdde25f3ce4951" ], - "markers": "python_version >= '3.6'", - "version": "==1.2.0" + "markers": "python_version >= '3.7'", + "version": "==1.3.0" }, "google-auth": { "hashes": [ @@ -426,11 +413,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:92a8b58ce734b2a4494878e0ecf7d79ccd7a128b5fc6014c401e0b61f006f0f6", - "sha256:b7cf7d3fef75f1e4c80a96ca660efbd51473d7e8f39b5ab9210febc7809012a4" + "sha256:899e2a40a8c4a1aec681feef45733de8a6c58f3f6a0dbed2eb6574b4387a77b6", + "sha256:951f0d8a5b7260e9db5e41d429285b5f451e928479f19d80818878527d36e95e" ], "markers": "python_version < '3.9'", - "version": "==4.10.0" + "version": "==4.10.1" }, "importlib-resources": { "hashes": [ @@ -910,6 +897,14 @@ ], "version": "==0.27" }, + "pyjwt": { + "hashes": [ + "sha256:b888b4d56f06f6dcd777210c334e69c737be74755d3e5e9ee3fe67dc18a0ee41", + "sha256:e0c4bb8d9f0af0c7f5b1ec4c5036309617d03d56932877f2f7a0beeb5318322f" + ], + "index": "pypi", + "version": "==2.3.0" + }, "pyparsing": { "hashes": [ "sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4", @@ -1084,10 +1079,10 @@ }, "sentry-sdk": { "hashes": [ - "sha256:2cec50166bcb67e1965f8073541b2321e3864cd6fd42a526bcde9f0c4e4cc3f8", - "sha256:7bbaa32bba806ec629962f207b597e86831c7ee2c1f287c21ba7de7fea9a9c46" + "sha256:141da032f0fa4c56f9af6b361fda57360af1789576285bd1944561f9c274f9c0", + "sha256:9aeff2a47f4038460296b920bf4d269284e8454e1c67547ee002ccafd9c2442b" ], - "version": "==1.5.2" + "version": "==1.5.3" }, "six": { "hashes": [ @@ -1154,11 +1149,11 @@ }, "thoth-common": { "hashes": [ - "sha256:09032ee49cc78e1a18276703f0f0a5f55bff889c33ca51863b3a89c9cb9120e6", - "sha256:c19bc2990393181d268a4ed6de350e903f99a779d3822120bda15e0c17e00006" + "sha256:4773480d24c72162be72d5777b5f69312f436316da45d83c27080243ffca9129", + "sha256:daae81830122b1f35ad5314cd5a7d9d7d86ed5fe77c39d01fc95623dd20248d6" ], "index": "pypi", - "version": "==0.33.1" + "version": "==0.34.0" }, "thoth-messaging": { "hashes": [ @@ -1490,11 +1485,11 @@ }, "hypothesis": { "hashes": [ - "sha256:b51d1cbfd5c5d38b59435f695a1ff790c5fa7600baa59eba202a7bf8498d9043", - "sha256:ce3961fff61e7353d022608788cbc9876c293d2468749eeba27511adc9565131" + "sha256:2b9c56faa067d660f0802679689f825bf142eec8261ab9e2e6ea916b1d8278a1", + "sha256:fa9f845b06199ea87e68c6da04a609ff46e381b5d542351184790d54eaca144c" ], "index": "pypi", - "version": "==6.35.0" + "version": "==6.36.0" }, "hypothesis-auto": { "hashes": [ @@ -1506,11 +1501,11 @@ }, "identify": { "hashes": [ - "sha256:8d80d877441073e7222ff272da45ca5ca1d901412790544b446e7d363ad5e9f0", - "sha256:ac9c919c8ac5b90d03f8e1bce6dedcbd8a3b2a59c46737c04ed852e2d307af29" + "sha256:6b4b5031f69c48bf93a646b90de9b381c6b5f560df4cbe0ed3cf7650ae741e4d", + "sha256:aa68609c7454dbcaae60a01ff6b8df1de9b39fe6e50b1f6107ec81dcda624aa6" ], "markers": "python_full_version >= '3.6.1'", - "version": "==2.4.3" + "version": "==2.4.4" }, "iniconfig": { "hashes": [ @@ -1674,11 +1669,11 @@ }, "pytest-timeout": { "hashes": [ - "sha256:e6f98b54dafde8d70e4088467ff621260b641eb64895c4195b6e5c8f45638112", - "sha256:fe9c3d5006c053bb9e062d60f641e6a76d6707aedb645350af9593e376fcc717" + "sha256:c07ca07404c612f8abbe22294b23c368e2e5104b521c1790195561f37e1ac3d9", + "sha256:f6f50101443ce70ad325ceb4473c4255e9d74e3c7cd0ef827309dfa4c0d975c6" ], "index": "pypi", - "version": "==2.0.2" + "version": "==2.1.0" }, "pyyaml": { "hashes": [ diff --git a/thoth/investigator/common.py b/thoth/investigator/common.py index 3d0b31b..a615730 100644 --- a/thoth/investigator/common.py +++ b/thoth/investigator/common.py @@ -22,9 +22,10 @@ from math import inf from asyncio import sleep import json - from typing import List, Tuple, Optional, Callable +from ogr.services.github import GithubService + from thoth.common import OpenShift from thoth.storages import GraphDatabase from thoth.messaging import ALL_MESSAGES @@ -307,3 +308,14 @@ async def _schedule_all_solvers( raise e return are_scheduled + + +# KP TD: download and load files from github for user-api request +# def _generate_advise_inputs_from_slug_and_env(github_project: GithubProject) -> dict: +# project = +# db.get_python_software_stack_dict_from_id() +# pass + +# def _generate_runtime_environment_from_slug_and_env(github_project: GithubProject) -> dict: +# db.get_runtime_environment_dict_from_id() +# pass diff --git a/thoth/investigator/github_service.py b/thoth/investigator/github_service.py new file mode 100644 index 0000000..03519ae --- /dev/null +++ b/thoth/investigator/github_service.py @@ -0,0 +1,204 @@ +from typing import Optional, Union, NamedTuple +import asyncio +import time +from datetime import datetime +import json + +import aiohttp +import jwt + +# this is an async wrapper for the GitHub API a more feature rich library is +# available here, but it is unclear whether this project is stable. +# https://github.com/brettcannon/gidgethub/issues + + +BASE_API_URL = "https://api.github.com" + + +class InstallationNotFoundError(Exception): + pass + + +class UnknownResponseCodeError(Exception): + pass + + +class AccessToken(NamedTuple): + token: str + expiration: Optional[Union[int, float]] = None # epoch time + + +class GithubAuth: + def __init__(self): + self._event_loop = asyncio.get_event_loop() + self._session = aiohttp.ClientSession( + loop=self._event_loop, headers={"Accept": "application/vnd.github.v3"} # current API version + ) + + def __del__(self): + self._event_loop.run_until_complete(self._session.close()) + + async def get_auth_token(self): + raise NotImplementedError() + + async def get_repository_token(self, namespace, repo): + raise NotImplementedError() + + +class GithubOAuth(GithubAuth): + def __init__(self, token: str): + self._token = AccessToken(token=token, expiration=None) + super().__init__() + + async def get_auth_token(self): + return self._token + + async def get_repository_token(self, _, unused): # noqa: F831 + return self._token + + +class GithubApp(GithubAuth): + def __init__(self, app_id, app_private_key_path: str): + self.app_private_key = self._get_app_private_key(app_private_key_path) + self.app_id = app_id + super().__init__() + + def _generate_jwt(self): + payload = { + # issued at time, 60 seconds in the past to allow for clock drift + "iat": int(time.time()) - 60, + # JWT expiration time (1 minute) + "exp": int(time.time()).timestamp() + 60, + # GitHub App's identifier + "iss": self.app_id, + } + + encrypted = jwt.encode(payload, self.app_private_key, algorithm="RS256") + if isinstance(encrypted, bytes): + encrypted = encrypted.decode("utf-8") + return encrypted + + @staticmethod + def _get_app_private_key(private_key_path) -> str: + """ + Get private key string from file. + Returns + str: PEM256 key found in private_key_path + Raises + FileNotFoundError + """ + with open(private_key_path, "r") as f: + return f.read() + + async def get_auth_token(self): + """ + Get App auth token. + Note + expires in 60s it is best to generate a new token for each request + """ + return AccessToken(token=self._generate_jwt(), expiration=int(time.time())) + + async def _get_repository_installation(self, namespace, repo): + """https://docs.github.com/en/rest/reference/apps#get-a-repository-installation-for-the-authenticated-app""" + async with self._session.get( + f"{BASE_API_URL}/app/{namespace}/{repo}/installation", + headers={"Authorization": f"Bearer {(await self.get_auth_token()).token}"}, + ) as r: + if r.status == 404: + raise InstallationNotFoundError(f"App not installed for {namespace}/{repo}.") + elif r.status == 200: + return r.json() + else: + raise UnknownResponseCodeError(f"GitHub API responded with status: {r.status}") + + async def get_repository_token(self, namespace, repo): + installation_info = await self._get_repository_installation(namespace, repo) + installation_id = installation_info["id"] + async with self._session.post( + f"{BASE_API_URL}/app/installations/{installation_id}/access_tokens", + headers={"Authorization": f"Bearer {(await self.get_auth_token()).token}"}, + ) as r: + if r.status == 401: # Unauthorized + pass + elif r.status == 403: # Forbidden + pass + elif r.status == 404: # NotFound + pass + elif r.status == 415: # UnsupportedMediaType + pass + elif r.status == 422: # UnprocessableEntity + pass + elif r.status == 201: # Successfully created + r_dict = await r.json() + AccessToken( + r_dict["token"], datetime.strptime(r_dict["expires_at"], "%Y-%m-%dT%H:%M:%S.%fZ").timestamp() + ) + + +class GithubService: + def __init__(self, token, app_id, app_private_key_path): + if app_private_key_path and app_id: + self._auth = GithubApp(app_id=app_id, app_private_key_path=app_private_key_path) + elif token: + self._auth = GithubOAuth(token=token) + + def get_project(self, namespace, repo): + return GithubProject(self, namespace, repo) + + +class GithubProject: + def __init__( + self, + github_service: GithubService, + namespace: str, + repo: str, + repository_id: Optional[int] = None, + ): + self._repository_id = repository_id # for cases where we already have the repo_id to minimize api calls + self._github_service = github_service + self.namespace = namespace + self.repo = repo + self._token: Optional[AccessToken] = None + + async def get_token(self) -> str: + if self._token is None: + self._token = await self._github_service._auth.get_repository_token(self.namespace, self.repo) + elif self._token.expiration is not None and self._token.expiration - time.time() < 0: # refresh token + self._token = await self._github_service._auth.get_repository_token(self.namespace, self.repo) + return self._token.token + + async def get_file(self, file_path: str, destination: str) -> str: + """ + Download GitHub file to given path + raises: + FileNotFoundError, TypeError + returns: + None + GitHubAPI docs: + https://docs.github.com/en/rest/reference/repos#get-repository-content + """ + async with self._github_service._auth._session.get( + f"{BASE_API_URL}/repos/{self.namespace}/{self.repo}/contents/{file_path}", + headers={"Authorization": f"token {await self.get_token()}"}, + ) as r: + if r.status == 404: + raise FileNotFoundError(f"{self.namespace}/{self.repo} does not have {file_path}") + elif r.status == 200: + r_dict = await r.json() + if type(r_dict) == list: + raise TypeError(f"{self.namespace}/{self.repo} has {file_path} but it is a directory") + if r_dict["type"] != "file": + raise TypeError(f"{self.namespace}/{self.repo} has {file_path} but it is a {r_dict['type']}") + download_url = r_dict["download_url"] + async with self._github_service._auth._session.get( + download_url, headers={"Authorization": f"token {await self.get_token()}"} + ) as r2: + if r2.status == 200: + return await r2.read() + else: + raise UnknownResponseCodeError( + f"Unknown response code, {r.status}, when downloading file from git" + ) + else: + raise UnknownResponseCodeError(f"github api responded with status code {r.status} which is not handled") + # TODO: handle codes 302, 403 diff --git a/thoth/investigator/solved_package/investigate_solved_package.py b/thoth/investigator/solved_package/investigate_solved_package.py index f385609..247740c 100644 --- a/thoth/investigator/solved_package/investigate_solved_package.py +++ b/thoth/investigator/solved_package/investigate_solved_package.py @@ -72,6 +72,40 @@ async def parse_solved_package_message( "SOLVER_NAME": solved_package.get("solver"), # We pass the solver name also. } + # KP: get all kebechet installation entries + + # KP: generate user-api request for each keb-installation + # turn runtime_environment id and software_stack id into JSON for request body (helper function) + # could this function be added to storages instead of here? + + # KP: add user-api authentication to config and deployment + + # KP: is "library_usage" important for adviser? This info is lost because we do not have the source code + + # KP: use thamos to submit advise. thamos.lib.advise + # advise + # api_client: ApiClient, (Needs to be generated) + # pipfile: str, (Generated from software_stack id) + # pipfile_lock: str, (Generated from software_stack id) + # *, + # recommendation_type: typing.Optional[str] = None, (may be present in runtime environment, otherwise need to get from .thoth.yaml) + # runtime_environment_name: typing.Optional[str] = None, (Generated from runtime_environment id) + # nowait: bool = True, + # origin: typing.Optional[str] = None, (something like github.com/{slug}) + # github_installation_id: typing.Optional[int] = None, (does this need set?) + # github_base_repo_url: typing.Optional[str] = None, (does this need set?) + # source_type: typing.Optional[ThothAdviserIntegrationEnum] = None, (should this be set to Kebechet?) + # justification: typing.Optional[Dict] = None, (add reason for running here) + # stack_info: typing.Optional[Dict] = None, (not sure what this is) + # kebechet_metadata: typing.Optional[Dict] = None, (this should contain info about the package and internal trigger see: https://github.com/thoth-station/messaging/blob/master/thoth/messaging/kebechet_run_url.py) + # KP: the required parameters are as follows + # recommendation_type (off the top of my head this will require getting .thoth.yaml from GitHub) + # source_type (not sure what value we should put here (Kebechet|None|new value)) + # origin (github.com/{slug}) + # token (from environment) + # application stack (must be generated from software_stack id) + # runtime environment (must be generated from runtime_environment id) + # We schedule Kebechet Administrator workflow here - workflow_id = await common.schedule_kebechet_administrator( openshift=openshift,