From 961176e0df4808bf059a1bf267b23ba991e6afbb Mon Sep 17 00:00:00 2001 From: Michael Skarbek Date: Mon, 10 Feb 2025 12:46:44 -0500 Subject: [PATCH] [COST-5886] update kafka status check (#5494) --- Pipfile | 1 - Pipfile.lock | 311 +++++++++++------------ koku/kafka_utils/test/test_utils.py | 24 +- koku/kafka_utils/utils.py | 25 +- koku/sources/kafka_listener.py | 3 - koku/sources/test/test_kafka_listener.py | 16 +- 6 files changed, 190 insertions(+), 190 deletions(-) diff --git a/Pipfile b/Pipfile index be02e56d5b..98b9fed865 100644 --- a/Pipfile +++ b/Pipfile @@ -36,7 +36,6 @@ gunicorn = "*" ibm-cloud-sdk-core = ">=3.5.2" ibm-platform-services = ">=0.17.8" jinjasql2 = "*" -kafka-python = ">=2.0.1" kombu = "<5.3" # https://issues.redhat.com/browse/COST-3997 msrestazure = "*" numpy = {version = "*", markers = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64'"} diff --git a/Pipfile.lock b/Pipfile.lock index 42293e0a4d..6cd986bc56 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "87d49f8ee5401932957699d19f063187e5f6e29c95f9a305f062d813b3b59a20" + "sha256": "c7b19ac33c63e49bd29d6e1a2c2d4d11df1130f126317ea740c3b51c90e4d18a" }, "pipfile-spec": 6, "requires": { @@ -188,11 +188,11 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:9c4c3dfa67aba55f6cd03769c441b21e6a369797fd6766e4b4c6b3399aae2735", - "sha256:b6e5afb3a2b1472c8db751a92eabf7834e5c7099f990c5e4b35f1f16b60bae64" + "sha256:1bd32405dacc920b42b83ba01644747ed77456a65760e285fbc47633ceddaf8b", + "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16" ], - "markers": "python_full_version >= '3.6.0'", - "version": "==4.13.0" + "markers": "python_full_version >= '3.7.0'", + "version": "==4.13.3" }, "billiard": { "hashes": [ @@ -592,21 +592,21 @@ }, "django": { "hashes": [ - "sha256:52ae8eacf635617c0f13b44f749e5ea13dc34262819b2cc8c8636abb08d82c4b", - "sha256:ba52eff7e228f1c775d5b0db2ba53d8c49d2f8bfe6ca0234df6b7dd12fb25b19" + "sha256:6c833be4b0ca614f0a919472a1028a3bbdeb6f056fa04023aeb923346ba2c306", + "sha256:a104e13f219fc55996a4e416ef7d18ab4eeb44e0aa95174c192f16cda9f94e75" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.2.18" + "version": "==4.2.19" }, "django-cors-headers": { "hashes": [ - "sha256:14d76b4b4c8d39375baeddd89e4f08899051eeaf177cb02a29bd6eae8cf63aa8", - "sha256:8edbc0497e611c24d5150e0055d3b178c6534b8ed826fb6f53b21c63f5d48ba3" + "sha256:6fdf31bf9c6d6448ba09ef57157db2268d515d94fc5c89a0a1028e1fc03ee52b", + "sha256:f1c125dcd58479fe7a67fe2499c16ee38b81b397463cf025f0e2c42937421070" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==4.6.0" + "version": "==4.7.0" }, "django-environ": { "hashes": [ @@ -947,14 +947,6 @@ "markers": "python_version >= '3.7'", "version": "==1.0.1" }, - "kafka-python": { - "hashes": [ - "sha256:04dfe7fea2b63726cd6f3e79a2d86e709d608d74406638c5da33a01d45a9d7e3", - "sha256:2d92418c7cb1c298fa6c7f0fb3519b520d0d7526ac6cb7ae2a4fc65a51a94b6e" - ], - "index": "pypi", - "version": "==2.0.2" - }, "kombu": { "hashes": [ "sha256:37cee3ee725f94ea8bb173eaab7c1760203ea53bbebae226328600f9d2799610", @@ -1258,11 +1250,11 @@ }, "oci": { "hashes": [ - "sha256:b3b1c744af964ab815eef8e68a1f2e1366c68e1be54ae8e7427a90929297f404", - "sha256:dbb577c04e8862b2e0f5f7d152c8e59066f821dba6bfddb8368cce761a609fd7" + "sha256:7a03b06b9323ae4c08127e45d2dedb88609ab37a02975761f34e952344c0b8de", + "sha256:b5d5a07ca18d15a230eb3bc15f89fd8cbe6523986600375aab48b64ee51dcf92" ], "index": "pypi", - "version": "==2.143.1" + "version": "==2.144.0" }, "packaging": { "hashes": [ @@ -1339,17 +1331,17 @@ }, "polars": { "hashes": [ - "sha256:063f8807f633f8fd15458a43971d930f6ee568b8e95936d7736c9054fc4f6f52", - "sha256:519863e0990e3323e7a32fc66bac3ad9da51938a1ffce6c09a92e0b1adb026a5", - "sha256:6bb0ba805defb05b76fdca392e48d84d1f16403de5be25d4dd8cdc7fccfd4251", - "sha256:7692d0fe0fb4faac18ef9423de55789e289f4d3f26d42519bd23ef8afb672d62", - "sha256:bbecddca35c57efde99070517db5d2c63d4c6d0e3c992123ba3be93e86e7bfac", - "sha256:c4517abb008af890e4ca8fb6bb0372868381017af0ecadf9d062e2f91f50b276", - "sha256:d9ce8e6f0d8140e67b0f7c276d22bb5f3345ce7412558643c8b5c270db254b64" + "sha256:5ee3cf3783205709ce31f070f2b4ee4296fec08f2c744a9c37acc7d360121022", + "sha256:6250f838b916fab23ccafe90928d7952afc328d316c956b42d152b20c86ffd9c", + "sha256:729e6be8a884812a206518195a2fb407b61962323886095ede1a2a934cdb1410", + "sha256:78b8bcd1735e9376815d117aeae49391441b2199b5a70a300669d692b34ec713", + "sha256:8d94ae25085d92de10d93ab6a06c94f8c911bd5d9c1ff17cd1073a9dca766029", + "sha256:94f25b4ef131da046d05b8235c5f29997630ee2125ebc0553b92258e88f7a8fa", + "sha256:cde8f56c408151ab9790c43485b90f690d5c198ce26ab38a845045c73c999325" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==1.21.0" + "version": "==1.22.0" }, "portalocker": { "hashes": [ @@ -1842,12 +1834,12 @@ }, "trino": { "hashes": [ - "sha256:23fb44d0bafb5175f1111fef5a2f0caf32c751c32a26c0126553777f91bd2f56", - "sha256:24ef06c231578cff55b966a585a177d847bcb77a02504ac9c96394676a884952" + "sha256:9954e59e32c5d562e1def5307d466a6cdd986b3c9ac6979d07d4fe023968da6b", + "sha256:f301f54900182fa8f07de9198a3f0cdfb68216db0c61558d324bf42f1582fe45" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==0.332.0" + "version": "==0.333.0" }, "typing-extensions": { "hashes": [ @@ -1923,12 +1915,12 @@ }, "whitenoise": { "hashes": [ - "sha256:486bd7267a375fa9650b136daaec156ac572971acc8bf99add90817a530dd1d4", - "sha256:df12dce147a043d1956d81d288c6f0044147c6d2ab9726e5772ac50fb45d2280" + "sha256:8c4a7c9d384694990c26f3047e118c691557481d624f069b7f7752a2f735d609", + "sha256:c8a489049b7ee9889617bb4c274a153f3d979e8f51d2efd0f5b403caf41c57df" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==6.8.2" + "version": "==6.9.0" }, "zstandard": { "hashes": [ @@ -2472,81 +2464,76 @@ }, "coverage": { "hashes": [ - "sha256:05fca8ba6a87aabdd2d30d0b6c838b50510b56cdcfc604d40760dae7153b73d9", - "sha256:0aa9692b4fdd83a4647eeb7db46410ea1322b5ed94cd1715ef09d1d5922ba87f", - "sha256:0c807ca74d5a5e64427c8805de15b9ca140bba13572d6d74e262f46f50b13273", - "sha256:0d7a2bf79378d8fb8afaa994f91bfd8215134f8631d27eba3e0e2c13546ce994", - "sha256:0f460286cb94036455e703c66988851d970fdfd8acc2a1122ab7f4f904e4029e", - "sha256:204a8238afe787323a8b47d8be4df89772d5c1e4651b9ffa808552bdf20e1d50", - "sha256:2396e8116db77789f819d2bc8a7e200232b7a282c66e0ae2d2cd84581a89757e", - "sha256:254f1a3b1eef5f7ed23ef265eaa89c65c8c5b6b257327c149db1ca9d4a35f25e", - "sha256:26bcf5c4df41cad1b19c84af71c22cbc9ea9a547fc973f1f2cc9a290002c8b3c", - "sha256:27c6e64726b307782fa5cbe531e7647aee385a29b2107cd87ba7c0105a5d3853", - "sha256:299e91b274c5c9cdb64cbdf1b3e4a8fe538a7a86acdd08fae52301b28ba297f8", - "sha256:2bcfa46d7709b5a7ffe089075799b902020b62e7ee56ebaed2f4bdac04c508d8", - "sha256:2ccf240eb719789cedbb9fd1338055de2761088202a9a0b73032857e53f612fe", - "sha256:32ee6d8491fcfc82652a37109f69dee9a830e9379166cb73c16d8dc5c2915165", - "sha256:3f7b444c42bbc533aaae6b5a2166fd1a797cdb5eb58ee51a92bee1eb94a1e1cb", - "sha256:457574f4599d2b00f7f637a0700a6422243b3565509457b2dbd3f50703e11f59", - "sha256:489a01f94aa581dbd961f306e37d75d4ba16104bbfa2b0edb21d29b73be83609", - "sha256:4bcc276261505d82f0ad426870c3b12cb177752834a633e737ec5ee79bbdff18", - "sha256:4e0de1e902669dccbf80b0415fb6b43d27edca2fbd48c74da378923b05316098", - "sha256:4e4630c26b6084c9b3cb53b15bd488f30ceb50b73c35c5ad7871b869cb7365fd", - "sha256:4eea95ef275de7abaef630c9b2c002ffbc01918b726a39f5a4353916ec72d2f3", - "sha256:507a20fc863cae1d5720797761b42d2d87a04b3e5aeb682ef3b7332e90598f43", - "sha256:54a5f0f43950a36312155dae55c505a76cd7f2b12d26abeebbe7a0b36dbc868d", - "sha256:55b201b97286cf61f5e76063f9e2a1d8d2972fc2fcfd2c1272530172fd28c359", - "sha256:59af35558ba08b758aec4d56182b222976330ef8d2feacbb93964f576a7e7a90", - "sha256:5c912978f7fbf47ef99cec50c4401340436d200d41d714c7a4766f377c5b7b78", - "sha256:656c82b8a0ead8bba147de9a89bda95064874c91a3ed43a00e687f23cc19d53a", - "sha256:6713ba4b4ebc330f3def51df1d5d38fad60b66720948112f114968feb52d3f99", - "sha256:675cefc4c06e3b4c876b85bfb7c59c5e2218167bbd4da5075cbe3b5790a28988", - "sha256:6f93531882a5f68c28090f901b1d135de61b56331bba82028489bc51bdd818d2", - "sha256:714f942b9c15c3a7a5fe6876ce30af831c2ad4ce902410b7466b662358c852c0", - "sha256:79109c70cc0882e4d2d002fe69a24aa504dec0cc17169b3c7f41a1d341a73694", - "sha256:7bbd8c8f1b115b892e34ba66a097b915d3871db7ce0e6b9901f462ff3a975377", - "sha256:7ed2f37cfce1ce101e6dffdfd1c99e729dd2ffc291d02d3e2d0af8b53d13840d", - "sha256:7fb105327c8f8f0682e29843e2ff96af9dcbe5bab8eeb4b398c6a33a16d80a23", - "sha256:89d76815a26197c858f53c7f6a656686ec392b25991f9e409bcef020cd532312", - "sha256:9a7cfb50515f87f7ed30bc882f68812fd98bc2852957df69f3003d22a2aa0abf", - "sha256:9e1747bab246d6ff2c4f28b4d186b205adced9f7bd9dc362051cc37c4a0c7bd6", - "sha256:9e80eba8801c386f72e0712a0453431259c45c3249f0009aff537a517b52942b", - "sha256:a01ec4af7dfeb96ff0078ad9a48810bb0cc8abcb0115180c6013a6b26237626c", - "sha256:a372c89c939d57abe09e08c0578c1d212e7a678135d53aa16eec4430adc5e690", - "sha256:a3b204c11e2b2d883946fe1d97f89403aa1811df28ce0447439178cc7463448a", - "sha256:a534738b47b0de1995f85f582d983d94031dffb48ab86c95bdf88dc62212142f", - "sha256:a5e37dc41d57ceba70956fa2fc5b63c26dba863c946ace9705f8eca99daecdc4", - "sha256:aa744da1820678b475e4ba3dfd994c321c5b13381d1041fe9c608620e6676e25", - "sha256:ab32947f481f7e8c763fa2c92fd9f44eeb143e7610c4ca9ecd6a36adab4081bd", - "sha256:abb02e2f5a3187b2ac4cd46b8ced85a0858230b577ccb2c62c81482ca7d18852", - "sha256:b330368cb99ef72fcd2dc3ed260adf67b31499584dc8a20225e85bfe6f6cfed0", - "sha256:bc67deb76bc3717f22e765ab3e07ee9c7a5e26b9019ca19a3b063d9f4b874244", - "sha256:c0b1818063dc9e9d838c09e3a473c1422f517889436dd980f5d721899e66f315", - "sha256:c56e097019e72c373bae32d946ecf9858fda841e48d82df7e81c63ac25554078", - "sha256:c7827a5bc7bdb197b9e066cdf650b2887597ad124dd99777332776f7b7c7d0d0", - "sha256:ccc2b70a7ed475c68ceb548bf69cec1e27305c1c2606a5eb7c3afff56a1b3b27", - "sha256:d37a84878285b903c0fe21ac8794c6dab58150e9359f1aaebbeddd6412d53132", - "sha256:e2f0280519e42b0a17550072861e0bc8a80a0870de260f9796157d3fca2733c5", - "sha256:e4ae5ac5e0d1e4edfc9b4b57b4cbecd5bc266a6915c500f358817a8496739247", - "sha256:e67926f51821b8e9deb6426ff3164870976fe414d033ad90ea75e7ed0c2e5022", - "sha256:e78b270eadb5702938c3dbe9367f878249b5ef9a2fcc5360ac7bff694310d17b", - "sha256:ea3c8f04b3e4af80e17bab607c386a830ffc2fb88a5484e1df756478cf70d1d3", - "sha256:ec22b5e7fe7a0fa8509181c4aac1db48f3dd4d3a566131b313d1efc102892c18", - "sha256:f4f620668dbc6f5e909a0946a877310fb3d57aea8198bde792aae369ee1c23b5", - "sha256:fd34e7b3405f0cc7ab03d54a334c17a9e802897580d964bd8c2001f4b9fd488f" + "sha256:050172741de03525290e67f0161ae5f7f387c88fca50d47fceb4724ceaa591d2", + "sha256:08e5fb93576a6b054d3d326242af5ef93daaac9bb52bc25f12ccbc3fa94227cd", + "sha256:09d03f48d9025b8a6a116cddcb6c7b8ce80e4fb4c31dd2e124a7c377036ad58e", + "sha256:0d03c9452d9d1ccfe5d3a5df0427705022a49b356ac212d529762eaea5ef97b4", + "sha256:13100f98497086b359bf56fc035a762c674de8ef526daa389ac8932cb9bff1e0", + "sha256:25575cd5a7d2acc46b42711e8aff826027c0e4f80fb38028a74f31ac22aae69d", + "sha256:27700d859be68e4fb2e7bf774cf49933dcac6f81a9bc4c13bd41735b8d26a53b", + "sha256:2c81e53782043b323bd34c7de711ed9b4673414eb517eaf35af92185b873839c", + "sha256:397489c611b76302dfa1d9ea079e138dddc4af80fc6819d5f5119ec8ca6c0e47", + "sha256:476f29a258b9cd153f2be5bf5f119d670d2806363595263917bddc167d6e5cce", + "sha256:4bda710139ea646890d1c000feb533caff86904a0e0638f85e967c28cb8eec50", + "sha256:4cf96beb05d004e4c51cd846fcdf9eee9eb2681518524b66b2e7610507944c2f", + "sha256:4f21e3617f48d683f30cf2a6c8b739c838e600cb1454fe6b2eb486ac2bce8fbd", + "sha256:5128f3ba694c0a1bde55fc480090392c336236c3e1a10dad40dc1ab17c7675ff", + "sha256:532fe139691af134aa8b54ed60dd3c806aa81312d93693bd2883c7b61592c840", + "sha256:5a3f7cbbcb4ad95067a6525f83a6fc78d9cbc1e70f8abaeeaeaa72ef34f48fc3", + "sha256:5b48db06f53d1864fea6dbd855e6d51d41c0f06c212c3004511c0bdc6847b297", + "sha256:5e7ac966ab110bd94ee844f2643f196d78fde1cd2450399116d3efdd706e19f5", + "sha256:5edc16712187139ab635a2e644cc41fc239bc6d245b16124045743130455c652", + "sha256:60d4ad09dfc8c36c4910685faafcb8044c84e4dae302e86c585b3e2e7778726c", + "sha256:61c834cbb80946d6ebfddd9b393a4c46bec92fcc0fa069321fcb8049117f76ea", + "sha256:6ba27a0375c5ef4d2a7712f829265102decd5ff78b96d342ac2fa555742c4f4f", + "sha256:6c96a142057d83ee993eaf71629ca3fb952cda8afa9a70af4132950c2bd3deb9", + "sha256:6d60577673ba48d8ae8e362e61fd4ad1a640293ffe8991d11c86f195479100b7", + "sha256:7eb0504bb307401fd08bc5163a351df301438b3beb88a4fa044681295bbefc67", + "sha256:8e433b6e3a834a43dae2889adc125f3fa4c66668df420d8e49bc4ee817dd7a70", + "sha256:8fa4fffd90ee92f62ff7404b4801b59e8ea8502e19c9bf2d3241ce745b52926c", + "sha256:90de4e9ca4489e823138bd13098af9ac8028cc029f33f60098b5c08c675c7bda", + "sha256:a165b09e7d5f685bf659063334a9a7b1a2d57b531753d3e04bd442b3cfe5845b", + "sha256:a46d56e99a31d858d6912d31ffa4ede6a325c86af13139539beefca10a1234ce", + "sha256:ac476e6d0128fb7919b3fae726de72b28b5c9644cb4b579e4a523d693187c551", + "sha256:ac5d92e2cc121a13270697e4cb37e1eb4511ac01d23fe1b6c097facc3b46489e", + "sha256:adc2d941c0381edfcf3897f94b9f41b1e504902fab78a04b1677f2f72afead4b", + "sha256:b6ff5be3b1853e0862da9d349fe87f869f68e63a25f7c37ce1130b321140f963", + "sha256:bb35ae9f134fbd9cf7302a9654d5a1e597c974202678082dcc569eb39a8cde03", + "sha256:be05bde21d5e6eefbc3a6de6b9bee2b47894b8945342e8663192809c4d1f08ce", + "sha256:c27df03730059118b8a923cfc8b84b7e9976742560af528242f201880879c1da", + "sha256:c7719a5e1dc93883a6b319bc0374ecd46fb6091ed659f3fbe281ab991634b9b0", + "sha256:c86f4c7a6d1a54a24d804d9684d96e36a62d3ef7c0d7745ae2ea39e3e0293251", + "sha256:ca95d40900cf614e07f00cee8c2fad0371df03ca4d7a80161d84be2ec132b7a4", + "sha256:cd4839813b09ab1dd1be1bbc74f9a7787615f931f83952b6a9af1b2d3f708bf7", + "sha256:db4b1a69976b1b02acda15937538a1d3fe10b185f9d99920b17a740a0a102e06", + "sha256:dbb1a822fd858d9853333a7c95d4e70dde9a79e65893138ce32c2ec6457d7a36", + "sha256:de6b079b39246a7da9a40cfa62d5766bd52b4b7a88cf5a82ec4c45bf6e152306", + "sha256:df6ff122a0a10a30121d9f0cb3fbd03a6fe05861e4ec47adb9f25e9245aabc19", + "sha256:e0b0f272901a5172090c0802053fbc503cdc3fa2612720d2669a98a7384a7bec", + "sha256:e2778be4f574b39ec9dcd9e5e13644f770351ee0990a0ecd27e364aba95af89b", + "sha256:e3b746fa0ffc5b6b8856529de487da8b9aeb4fb394bb58de6502ef45f3434f12", + "sha256:e642e6a46a04e992ebfdabed79e46f478ec60e2c528e1e1a074d63800eda4286", + "sha256:eafea49da254a8289bed3fab960f808b322eda5577cb17a3733014928bbfbebd", + "sha256:f0f334ae844675420164175bf32b04e18a81fe57ad8eb7e0cfd4689d681ffed7", + "sha256:f382004fa4c93c01016d9226b9d696a08c53f6818b7ad59b4e96cb67e863353a", + "sha256:f4679fcc9eb9004fdd1b00231ef1ec7167168071bebc4d66327e28c1979b4449", + "sha256:fd2fffc8ce8692ce540103dff26279d2af22d424516ddebe2d7e4d6dbb3816b2", + "sha256:ff136607689c1c87f43d24203b6d2055b42030f352d5176f9c8b204d4235ef27", + "sha256:ff52b4e2ac0080c96e506819586c4b16cdbf46724bda90d308a7330a73cc8521", + "sha256:ff562952f15eff27247a4c4b03e45ce8a82e3fb197de6a7c54080f9d4ba07845" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==7.6.10" + "version": "==7.6.11" }, "crc-bonfire": { "hashes": [ - "sha256:29c537d1bab36e88ee93d01bb3ec8cc49f6df2d23003537eb61d0f5ea798f1cb", - "sha256:d89acb2d2724ae15d1e42b64dceb072c20ba8a0dc3ef5f45aec8ae964f6f5d44" + "sha256:af18ace5c36716e74105b3e0026391f829d8269261ec14b16e0db69f66e48791", + "sha256:b81b06d0ce7b142a112418a24a1a4a3bc8e2b6bb573aa7c04bc9dcd60d673fa9" ], "index": "pypi", "markers": "python_version >= '3.6'", - "version": "==6.0.2" + "version": "==6.0.3" }, "cryptography": { "hashes": [ @@ -2631,12 +2618,12 @@ }, "django": { "hashes": [ - "sha256:52ae8eacf635617c0f13b44f749e5ea13dc34262819b2cc8c8636abb08d82c4b", - "sha256:ba52eff7e228f1c775d5b0db2ba53d8c49d2f8bfe6ca0234df6b7dd12fb25b19" + "sha256:6c833be4b0ca614f0a919472a1028a3bbdeb6f056fa04023aeb923346ba2c306", + "sha256:a104e13f219fc55996a4e416ef7d18ab4eeb44e0aa95174c192f16cda9f94e75" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.2.18" + "version": "==4.2.19" }, "django-cprofile-middleware": { "hashes": [ @@ -2689,59 +2676,59 @@ }, "fonttools": { "hashes": [ - "sha256:01ea3901b0802fc5f9e854f5aeb5bc27770dd9dd24c28df8f74ba90f8b3f5915", - "sha256:02c41322e5bdcb484b61b776fcea150215c83619b39c96aa0b44d4fd87bb5574", - "sha256:03c2b50b54e6e8b3564b232e57e8f58be217cf441cf0155745d9e44a76f9c30f", - "sha256:07636dae94f7fe88561f9da7a46b13d8e3f529f87fdb221b11d85f91eabceeb7", - "sha256:0f38bfb6b7a39c4162c3eb0820a0bdf8e3bdd125cd54e10ba242397d15e32439", - "sha256:1e10efc8ee10d6f1fe2931d41bccc90cd4b872f2ee4ff21f2231a2c293b2dbf8", - "sha256:2232012a1502b2b8ab4c6bc1d3524bfe90238c0c1a50ac94a0a2085aa87a58a5", - "sha256:243cbfc0b7cb1c307af40e321f8343a48d0a080bc1f9466cf2b5468f776ef108", - "sha256:285c1ac10c160fbdff6d05358230e66c4f98cbbf271f3ec7eb34e967771543e8", - "sha256:302e1003a760b222f711d5ba6d1ad7fd5f7f713eb872cd6a3eb44390bc9770af", - "sha256:332883b6280b9d90d2ba7e9e81be77cf2ace696161e60cdcf40cfcd2b3ed06fa", - "sha256:3461347016c94cb42b36caa907e11565878c4c2c375604f3651d11dc06d1ab3e", - "sha256:3d20f152de7625a0008ba1513f126daaaa0de3b4b9030aa72dd5c27294992260", - "sha256:450c354c04a6e12a3db968e915fe05730f79ff3d39560947ef8ee6eaa2ab2212", - "sha256:466a78984f0572305c3c48377f4e3f7f4e909f1209f45ef8e7041d5c8a744a56", - "sha256:4dfae7c94987149bdaa0388e6c937566aa398fa0eec973b17952350a069cff4e", - "sha256:54d481d456dcd59af25d4a9c56b2c4c3f20e9620b261b84144e5950f33e8df17", - "sha256:604c805b41241b4880e2dc86cf2d4754c06777371c8299799ac88d836cb18c3b", - "sha256:63403ee0f2fa4e1de28e539f8c24f2bdca1d8ecb503fa9ea2d231d9f1e729809", - "sha256:65cb8f97eed7906dcf19bc2736b70c6239e9d7e77aad7c6110ba7239ae082e81", - "sha256:67df1c3935838fb9e56f227d7f506c9043b149a4a3b667bef17929c7a1114d19", - "sha256:6b8d7c149d47b47de7ec81763396c8266e5ebe2e0b14aa9c3ccf29e52260ab2f", - "sha256:708cb17b2590b7f6c6854999df0039ff1140dda9e6f56d67c3599ba6f968fab5", - "sha256:8abd135e427d88e461a4833c03cf96cfb9028c78c15d58123291f22398e25492", - "sha256:9164f44add0acec0f12fce682824c040dc52e483bfe3838c37142897150c8364", - "sha256:95f5a1d4432b3cea6571f5ce4f4e9b25bf36efbd61c32f4f90130a690925d6ee", - "sha256:9b5f05ef72e846e9f49ccdd74b9da4309901a4248434c63c1ee9321adcb51d65", - "sha256:9b6fcff4dc755b32faff955d989ee26394ddad3a90ea7d558db17a4633c8390c", - "sha256:a0fe12f06169af2fdc642d26a8df53e40adc3beedbd6ffedb19f1c5397b63afd", - "sha256:a19059aa892676822c1f05cb5a67296ecdfeb267fe7c47d4758f3e8e942c2b2a", - "sha256:a7230f7590f9570d26ee903b6a4540274494e200fae978df0d9325b7b9144529", - "sha256:acfec948de41cd5e640d5c15d0200e8b8e7c5c6bb82afe1ca095cbc4af1188ee", - "sha256:b99d4fd2b6d0a00c7336c8363fccc7a11eccef4b17393af75ca6e77cf93ff413", - "sha256:b9f9fce3c9b2196e162182ec5db8af8eb3acd0d76c2eafe9fdba5f370044e556", - "sha256:ba45b637da80a262b55b7657aec68da2ac54b8ae7891cd977a5dbe5fd26db429", - "sha256:bf4b5b3496ddfdd4e57112e77ec51f1ab388d35ac17322c1248addb2eb0d429a", - "sha256:c96f2506ce1a0beeaa9595f9a8b7446477eb133f40c0e41fc078744c28149f80", - "sha256:cb121d6dd34625cece32234a5fa0359475bb118838b6b4295ffdb13b935edb04", - "sha256:ccf8ae02918f431953d338db4d0a675a395faf82bab3a76025582cf32a2f3b7b", - "sha256:cfe9cf30f391a0f2875247a3e5e44d8dcb61596e5cf89b360cdffec8a80e9961", - "sha256:d11600f5343092697d7434f3bf77a393c7ae74be206fe30e577b9a195fd53165", - "sha256:d2248ebfbcea0d0b3cb459d76a9f67f2eadc10ec0d07e9cadab8777d3f016bf2", - "sha256:d39f0c977639be0f9f5505d4c7c478236737f960c567a35f058649c056e41434", - "sha256:d5a3ff5bb95fd5a3962b2754f8435e6d930c84fc9e9921c51e802dddf40acd56", - "sha256:d637e4d33e46619c79d1a6c725f74d71b574cd15fb5bbb9b6f3eba8f28363573", - "sha256:de78d6d0dbe32561ce059265437021f4746e56073c4799f0f1095828ae7232bd", - "sha256:e72a7816ff8a759be9ca36ca46934f8ccf4383711ef597d9240306fe1878cb8d", - "sha256:edcffaeadba9a334c1c3866e275d7dd495465e7dbd296f688901bdbd71758113", - "sha256:f089e8da0990cfe2d67e81d9cf581ff372b48dc5acf2782701844211cd1f0eb3", - "sha256:f971aa5f50c22dc4b63a891503624ae2c77330429b34ead32f23c2260c5618cd" + "sha256:003548eadd674175510773f73fb2060bb46adb77c94854af3e0cc5bc70260049", + "sha256:0073b62c3438cf0058488c002ea90489e8801d3a7af5ce5f7c05c105bee815c3", + "sha256:1088182f68c303b50ca4dc0c82d42083d176cba37af1937e1a976a31149d4d14", + "sha256:133bedb9a5c6376ad43e6518b7e2cd2f866a05b1998f14842631d5feb36b5786", + "sha256:14a3e3e6b211660db54ca1ef7006401e4a694e53ffd4553ab9bc87ead01d0f05", + "sha256:17f39313b649037f6c800209984a11fc256a6137cbe5487091c6c7187cae4685", + "sha256:193b86e9f769320bc98ffdb42accafb5d0c8c49bd62884f1c0702bc598b3f0a2", + "sha256:2d351275f73ebdd81dd5b09a8b8dac7a30f29a279d41e1c1192aedf1b6dced40", + "sha256:300c310bb725b2bdb4f5fc7e148e190bd69f01925c7ab437b9c0ca3e1c7cd9ba", + "sha256:331954d002dbf5e704c7f3756028e21db07097c19722569983ba4d74df014000", + "sha256:38b947de71748bab150259ee05a775e8a0635891568e9fdb3cdd7d0e0004e62f", + "sha256:3cf4f8d2a30b454ac682e12c61831dcb174950c406011418e739de592bbf8f76", + "sha256:3fd3fccb7b9adaaecfa79ad51b759f2123e1aba97f857936ce044d4f029abd71", + "sha256:442ad4122468d0e47d83bc59d0e91b474593a8c813839e1872e47c7a0cb53b10", + "sha256:47b5e4680002ae1756d3ae3b6114e20aaee6cc5c69d1e5911f5ffffd3ee46c6b", + "sha256:53f5e9767978a4daf46f28e09dbeb7d010319924ae622f7b56174b777258e5ba", + "sha256:62b4c6802fa28e14dba010e75190e0e6228513573f1eeae57b11aa1a39b7e5b1", + "sha256:62cc1253827d1e500fde9dbe981219fea4eb000fd63402283472d38e7d8aa1c6", + "sha256:654ac4583e2d7c62aebc6fc6a4c6736f078f50300e18aa105d87ce8925cfac31", + "sha256:661a8995d11e6e4914a44ca7d52d1286e2d9b154f685a4d1f69add8418961563", + "sha256:6c1d38642ca2dddc7ae992ef5d026e5061a84f10ff2b906be5680ab089f55bb8", + "sha256:6e81c1cc80c1d8bf071356cc3e0e25071fbba1c75afc48d41b26048980b3c771", + "sha256:705837eae384fe21cee5e5746fd4f4b2f06f87544fa60f60740007e0aa600311", + "sha256:7ef04bc7827adb7532be3d14462390dd71287644516af3f1e67f1e6ff9c6d6df", + "sha256:86b2a1013ef7a64d2e94606632683f07712045ed86d937c11ef4dde97319c086", + "sha256:8d1613abd5af2f93c05867b3a3759a56e8bf97eb79b1da76b2bc10892f96ff16", + "sha256:965d0209e6dbdb9416100123b6709cb13f5232e2d52d17ed37f9df0cc31e2b35", + "sha256:96a4271f63a615bcb902b9f56de00ea225d6896052c49f20d0c91e9f43529a29", + "sha256:9d94449ad0a5f2a8bf5d2f8d71d65088aee48adbe45f3c5f8e00e3ad861ed81a", + "sha256:9da650cb29bc098b8cfd15ef09009c914b35c7986c8fa9f08b51108b7bc393b4", + "sha256:a05d1f07eb0a7d755fbe01fee1fd255c3a4d3730130cf1bfefb682d18fd2fcea", + "sha256:a114d1567e1a1586b7e9e7fc2ff686ca542a82769a296cef131e4c4af51e58f4", + "sha256:a1af375734018951c31c0737d04a9d5fd0a353a0253db5fbed2ccd44eac62d8c", + "sha256:b23d30a2c0b992fb1c4f8ac9bfde44b5586d23457759b6cf9a787f1a35179ee0", + "sha256:bc871904a53a9d4d908673c6faa15689874af1c7c5ac403a8e12d967ebd0c0dc", + "sha256:bce60f9a977c9d3d51de475af3f3581d9b36952e1f8fc19a1f2254f1dda7ce9c", + "sha256:bd9825822e7bb243f285013e653f6741954d8147427aaa0324a862cdbf4cbf62", + "sha256:ca7962e8e5fc047cc4e59389959843aafbf7445b6c08c20d883e60ced46370a5", + "sha256:d0cb73ccf7f6d7ca8d0bc7ea8ac0a5b84969a41c56ac3ac3422a24df2680546f", + "sha256:d54a45d30251f1d729e69e5b675f9a08b7da413391a1227781e2a297fa37f6d2", + "sha256:d6ca96d1b61a707ba01a43318c9c40aaf11a5a568d1e61146fafa6ab20890793", + "sha256:d6f195c14c01bd057bc9b4f70756b510e009c83c5ea67b25ced3e2c38e6ee6e9", + "sha256:e2cad98c94833465bcf28f51c248aaf07ca022efc6a3eba750ad9c1e0256d278", + "sha256:e2e993e8db36306cc3f1734edc8ea67906c55f98683d6fd34c3fc5593fdbba4c", + "sha256:e9270505a19361e81eecdbc2c251ad1e1a9a9c2ad75fa022ccdee533f55535dc", + "sha256:f20e2c0dfab82983a90f3d00703ac0960412036153e5023eed2b4641d7d5e692", + "sha256:f36a0868f47b7566237640c026c65a86d09a3d9ca5df1cd039e30a1da73098a0", + "sha256:f59746f7953f69cc3290ce2f971ab01056e55ddd0fb8b792c31a8acd7fee2d28", + "sha256:fa760e5fe8b50cbc2d71884a1eff2ed2b95a005f02dda2fa431560db0ddd927f", + "sha256:ffda9b8cd9cb8b301cae2602ec62375b59e2e2108a117746f12215145e3f786c" ], "markers": "python_version >= '3.8'", - "version": "==4.55.8" + "version": "==4.56.0" }, "google-api-core": { "extras": [ @@ -2935,11 +2922,11 @@ }, "identify": { "hashes": [ - "sha256:7bec12768ed44ea4761efb47806f0a41f86e7c0a5fdf5950d4648c90eca7e251", - "sha256:cbd1810bce79f8b671ecb20f53ee0ae8e86ae84b557de31d89709dc2a48ba881" + "sha256:155931cb617a401807b09ecec6635d6c692d180090a1cedca8ef7d58ba5b6aa0", + "sha256:3fa266b42eba321ee0b2bb0936a6a6b9e36a1351cbb69055b3082f4193035684" ], "markers": "python_version >= '3.9'", - "version": "==2.6.6" + "version": "==2.6.7" }, "idna": { "hashes": [ @@ -3393,11 +3380,11 @@ }, "oci": { "hashes": [ - "sha256:b3b1c744af964ab815eef8e68a1f2e1366c68e1be54ae8e7427a90929297f404", - "sha256:dbb577c04e8862b2e0f5f7d152c8e59066f821dba6bfddb8368cce761a609fd7" + "sha256:7a03b06b9323ae4c08127e45d2dedb88609ab37a02975761f34e952344c0b8de", + "sha256:b5d5a07ca18d15a230eb3bc15f89fd8cbe6523986600375aab48b64ee51dcf92" ], "index": "pypi", - "version": "==2.143.1" + "version": "==2.144.0" }, "ocviapy": { "hashes": [ diff --git a/koku/kafka_utils/test/test_utils.py b/koku/kafka_utils/test/test_utils.py index f4918371bd..62e9aa8eb8 100644 --- a/koku/kafka_utils/test/test_utils.py +++ b/koku/kafka_utils/test/test_utils.py @@ -15,14 +15,14 @@ class KafkaUtilsTest(TestCase): def test_check_kafka_connection(self): """Test check kafka connections.""" - with patch("kafka.BrokerConnection.connect_blocking", return_value=False): + with patch("kafka_utils.utils.get_admin_client") as mock_client: + mock_client.return_value.list_topics.return_value.topics = [] result = utils.check_kafka_connection() self.assertFalse(result) - with patch("kafka.BrokerConnection.connect_blocking", return_value=True): - with patch("kafka.BrokerConnection.close") as mock_close: - result = utils.check_kafka_connection() - mock_close.assert_called() - self.assertTrue(result) + with patch("kafka_utils.utils.get_admin_client") as mock_client: + mock_client.return_value.list_topics.return_value.topics = [1] + result = utils.check_kafka_connection() + self.assertTrue(result) @patch("time.sleep", side_effect=None) @patch("kafka_utils.utils.check_kafka_connection", side_effect=[bool(0), bool(1)]) @@ -78,3 +78,15 @@ def test_producer_singleton(self): p1 = utils.ProducerSingleton({}) p2 = utils.ProducerSingleton({}) self.assertEqual(id(p1), id(p2)) + + +class AdminClientSingletonTest(TestCase): + def test_producer_singleton(self): + """Tests that the ID of two created ProducerSingletons are in fact the same whereas two Producers are not.""" + # no provided bootstrap.servers create a producer that doesn't connect to anything. + pfake = utils.AdminClient({}) + pfake2 = utils.AdminClient({}) + self.assertNotEqual(id(pfake), id(pfake2)) + p1 = utils.AdminClientSingleton({}) + p2 = utils.AdminClientSingleton({}) + self.assertEqual(id(p1), id(p2)) diff --git a/koku/kafka_utils/utils.py b/koku/kafka_utils/utils.py index cde726519f..19ce345bfa 100644 --- a/koku/kafka_utils/utils.py +++ b/koku/kafka_utils/utils.py @@ -5,18 +5,16 @@ """Common utility functions for Kafka implementations.""" import logging import random -import socket import time from confluent_kafka import Consumer from confluent_kafka import Producer -from kafka import BrokerConnection +from confluent_kafka.admin import AdminClient from koku.configurator import CONFIGURATOR from masu.prometheus_stats import KAFKA_CONNECTION_ERRORS_COUNTER from masu.util.common import SingletonMeta - LOG = logging.getLogger(__name__) UPLOAD_TOPIC = CONFIGURATOR.get_kafka_topic("platform.upload.announce") VALIDATION_TOPIC = CONFIGURATOR.get_kafka_topic("platform.upload.validation") @@ -26,6 +24,10 @@ SOURCES_TOPIC = CONFIGURATOR.get_kafka_topic("platform.sources.event-stream") +class AdminClientSingleton(AdminClient, metaclass=SingletonMeta): + """Creates a singleton instance of a Kafka Producer""" + + class ProducerSingleton(Producer, metaclass=SingletonMeta): """Creates a singleton instance of a Kafka Producer""" @@ -68,6 +70,11 @@ def get_consumer(conf_settings): # pragma: no cover return Consumer(conf, logger=LOG) +def get_admin_client(): # pragma: no cover + conf = _get_managed_kafka_config() + return AdminClientSingleton(conf) + + def _get_producer_config(conf_settings): # pragma: no cover """Return Kafka Producer config""" producer_conf = {} @@ -102,14 +109,10 @@ def backoff(interval, maximum=120): def check_kafka_connection(): """Check connectability of Kafka Broker.""" - for broker in CONFIGURATOR.get_kafka_broker_list(): - host, port = broker.split(":") - conn = BrokerConnection(host, int(port), socket.AF_UNSPEC) - connected = conn.connect_blocking(timeout=1) - if connected: - conn.close() - break - return connected + client = get_admin_client() + topics = client.list_topics().topics + # if there is a list of topics, then we've successfully connected to kafka + return len(topics) > 0 def is_kafka_connected(): diff --git a/koku/sources/kafka_listener.py b/koku/sources/kafka_listener.py index 0891f05dc2..a1921b9512 100644 --- a/koku/sources/kafka_listener.py +++ b/koku/sources/kafka_listener.py @@ -20,7 +20,6 @@ from django.db import transaction from django.db.models.signals import post_save from django.dispatch import receiver -from kafka.errors import KafkaError from rest_framework.exceptions import ValidationError from api.provider.models import Sources @@ -304,8 +303,6 @@ def listen_for_messages(kaf_msg, consumer, application_source_id): # noqa: C901 else: consumer.commit() - except KafkaError as error: - LOG.error(f"[listen_for_messages] Kafka error encountered: {type(error).__name__}: {error}", exc_info=True) except Exception as error: LOG.error(f"[listen_for_messages] UNKNOWN error encountered: {type(error).__name__}: {error}", exc_info=True) diff --git a/koku/sources/test/test_kafka_listener.py b/koku/sources/test/test_kafka_listener.py index d71e8bd45e..9d7091ad16 100644 --- a/koku/sources/test/test_kafka_listener.py +++ b/koku/sources/test/test_kafka_listener.py @@ -9,6 +9,8 @@ from uuid import uuid4 import requests_mock +from confluent_kafka import KafkaError +from confluent_kafka import KafkaException from django.db import IntegrityError from django.db import InterfaceError from django.db import OperationalError @@ -16,7 +18,6 @@ from django.forms.models import model_to_dict from django.test.utils import override_settings from faker import Faker -from kafka.errors import KafkaError from rest_framework.exceptions import ValidationError import sources.kafka_listener as source_integration @@ -60,7 +61,6 @@ from sources.test.test_sources_http_client import MOCK_PREFIX from sources.test.test_sources_http_client import MOCK_URL - faker = Faker() FAKE_AWS_ARN = "arn:aws:iam::111111111111:role/CostManagement" FAKE_EXTERNAL_ID = str(uuid4()) @@ -106,7 +106,7 @@ def seek(self, topic_partition): def getone(self): for msg in self.preloaded_messages: return msg - raise KafkaError("Closing Mock Consumer") + raise KafkaException(KafkaError._PARTITION_EOF) def __aiter__(self): return self @@ -955,8 +955,9 @@ def test_storage_callback_update(self): local_source = Sources(**self.aws_local_source, koku_uuid=uuid, pending_update=True) local_source.save() - with patch("sources.kafka_listener.execute_process_queue"), patch( - "sources.storage.screen_and_build_provider_sync_create_event", return_value=False + with ( + patch("sources.kafka_listener.execute_process_queue"), + patch("sources.storage.screen_and_build_provider_sync_create_event", return_value=False), ): storage_callback("", local_source) _, msg = PROCESS_QUEUE.get_nowait() @@ -968,8 +969,9 @@ def test_storage_callback_update_and_delete(self): local_source = Sources(**self.aws_local_source, koku_uuid=uuid, pending_update=True, pending_delete=True) local_source.save() - with patch("sources.kafka_listener.execute_process_queue"), patch( - "sources.storage.screen_and_build_provider_sync_create_event", return_value=False + with ( + patch("sources.kafka_listener.execute_process_queue"), + patch("sources.storage.screen_and_build_provider_sync_create_event", return_value=False), ): storage_callback("", local_source) _, msg = PROCESS_QUEUE.get_nowait()