From 8e03bb5722627901acaf8037793c5680fc6d81a2 Mon Sep 17 00:00:00 2001 From: doughon <76968796+doughon@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:42:08 -0700 Subject: [PATCH] Gdxdsd 7170 python 3.8 audit (#484) * Updated the virtual environment to python 3.8 * Changed prompts to use input() or getpass() * Installed pandas * Installed SQLAlchemy * Changed to using dataframes to process the data * Results printed to console are now in csv format * Added optional flag to write results to console or file * Updated readme and help info * Removed extra quote escaping * Converted NaN to blank values * Specified SQLAlchemy version to 1.4.* * Create embed id column if missing for all users * Fixed Looker API3 key typo --- operations/auditing/Pipfile | 4 +- operations/auditing/Pipfile.lock | 345 ++++++++++++++++++++--- operations/auditing/README.md | 17 +- operations/auditing/createAuditReport.py | 170 ++++++----- 4 files changed, 420 insertions(+), 116 deletions(-) diff --git a/operations/auditing/Pipfile b/operations/auditing/Pipfile index 7fcaa407..444f2b1c 100644 --- a/operations/auditing/Pipfile +++ b/operations/auditing/Pipfile @@ -9,6 +9,8 @@ verify_ssl = true psycopg2-binary = "*" httplib2 = "*" argparse = "*" +pandas = "*" +sqlalchemy = "1.4.*" [requires] -python_version = "2.7" +python_version = "3.8" diff --git a/operations/auditing/Pipfile.lock b/operations/auditing/Pipfile.lock index 31682bb3..5fa64c9f 100644 --- a/operations/auditing/Pipfile.lock +++ b/operations/auditing/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "a3f3d9d613988c964893df3b0fa74ef1a55c0239d7a7c6d8ae527de0acff81f6" + "sha256": "dc5b1600e9935bc2c69f0cc97fd8e24249e681285636998a18c44a4b2214d255" }, "pipfile-spec": 6, "requires": { - "python_version": "2.7" + "python_version": "3.8" }, "sources": [ { @@ -24,58 +24,321 @@ "index": "pypi", "version": "==1.4.0" }, + "greenlet": { + "hashes": [ + "sha256:01059afb9b178606b4b6e92c3e710ea1635597c3537e44da69f4531e111dd5e9", + "sha256:037d9ac99540ace9424cb9ea89f0accfaff4316f149520b4ae293eebc5bded17", + "sha256:0e49a65d25d7350cca2da15aac31b6f67a43d867448babf997fe83c7505f57bc", + "sha256:13ff8c8e54a10472ce3b2a2da007f915175192f18e6495bad50486e87c7f6637", + "sha256:1544b8dd090b494c55e60c4ff46e238be44fdc472d2589e943c241e0169bcea2", + "sha256:184258372ae9e1e9bddce6f187967f2e08ecd16906557c4320e3ba88a93438c3", + "sha256:1ddc7bcedeb47187be74208bc652d63d6b20cb24f4e596bd356092d8000da6d6", + "sha256:221169d31cada333a0c7fd087b957c8f431c1dba202c3a58cf5a3583ed973e9b", + "sha256:243a223c96a4246f8a30ea470c440fe9db1f5e444941ee3c3cd79df119b8eebf", + "sha256:24fc216ec7c8be9becba8b64a98a78f9cd057fd2dc75ae952ca94ed8a893bf27", + "sha256:2651dfb006f391bcb240635079a68a261b227a10a08af6349cba834a2141efa1", + "sha256:26811df4dc81271033a7836bc20d12cd30938e6bd2e9437f56fa03da81b0f8fc", + "sha256:26d9c1c4f1748ccac0bae1dbb465fb1a795a75aba8af8ca871503019f4285e2a", + "sha256:28fe80a3eb673b2d5cc3b12eea468a5e5f4603c26aa34d88bf61bba82ceb2f9b", + "sha256:2cd8518eade968bc52262d8c46727cfc0826ff4d552cf0430b8d65aaf50bb91d", + "sha256:2d004db911ed7b6218ec5c5bfe4cf70ae8aa2223dffbb5b3c69e342bb253cb28", + "sha256:3d07c28b85b350564bdff9f51c1c5007dfb2f389385d1bc23288de51134ca303", + "sha256:3e7e6ef1737a819819b1163116ad4b48d06cfdd40352d813bb14436024fcda99", + "sha256:44151d7b81b9391ed759a2f2865bbe623ef00d648fed59363be2bbbd5154656f", + "sha256:44cd313629ded43bb3b98737bba2f3e2c2c8679b55ea29ed73daea6b755fe8e7", + "sha256:4a3dae7492d16e85ea6045fd11cb8e782b63eac8c8d520c3a92c02ac4573b0a6", + "sha256:4b5ea3664eed571779403858d7cd0a9b0ebf50d57d2cdeafc7748e09ef8cd81a", + "sha256:4c3446937be153718250fe421da548f973124189f18fe4575a0510b5c928f0cc", + "sha256:5415b9494ff6240b09af06b91a375731febe0090218e2898d2b85f9b92abcda0", + "sha256:5fd6e94593f6f9714dbad1aaba734b5ec04593374fa6638df61592055868f8b8", + "sha256:619935a44f414274a2c08c9e74611965650b730eb4efe4b2270f91df5e4adf9a", + "sha256:655b21ffd37a96b1e78cc48bf254f5ea4b5b85efaf9e9e2a526b3c9309d660ca", + "sha256:665b21e95bc0fce5cab03b2e1d90ba9c66c510f1bb5fdc864f3a377d0f553f6b", + "sha256:6a4bf607f690f7987ab3291406e012cd8591a4f77aa54f29b890f9c331e84989", + "sha256:6cea1cca3be76c9483282dc7760ea1cc08a6ecec1f0b6ca0a94ea0d17432da19", + "sha256:713d450cf8e61854de9420fb7eea8ad228df4e27e7d4ed465de98c955d2b3fa6", + "sha256:726377bd60081172685c0ff46afbc600d064f01053190e4450857483c4d44484", + "sha256:76b3e3976d2a452cba7aa9e453498ac72240d43030fdc6d538a72b87eaff52fd", + "sha256:76dc19e660baea5c38e949455c1181bc018893f25372d10ffe24b3ed7341fb25", + "sha256:76e5064fd8e94c3f74d9fd69b02d99e3cdb8fc286ed49a1f10b256e59d0d3a0b", + "sha256:7f346d24d74c00b6730440f5eb8ec3fe5774ca8d1c9574e8e57c8671bb51b910", + "sha256:81eeec4403a7d7684b5812a8aaa626fa23b7d0848edb3a28d2eb3220daddcbd0", + "sha256:90b5bbf05fe3d3ef697103850c2ce3374558f6fe40fd57c9fac1bf14903f50a5", + "sha256:9730929375021ec90f6447bff4f7f5508faef1c02f399a1953870cdb78e0c345", + "sha256:9eb4a1d7399b9f3c7ac68ae6baa6be5f9195d1d08c9ddc45ad559aa6b556bce6", + "sha256:a0409bc18a9f85321399c29baf93545152d74a49d92f2f55302f122007cfda00", + "sha256:a22f4e26400f7f48faef2d69c20dc055a1f3043d330923f9abe08ea0aecc44df", + "sha256:a53dfe8f82b715319e9953330fa5c8708b610d48b5c59f1316337302af5c0811", + "sha256:a771dc64fa44ebe58d65768d869fcfb9060169d203446c1d446e844b62bdfdca", + "sha256:a814dc3100e8a046ff48faeaa909e80cdb358411a3d6dd5293158425c684eda8", + "sha256:a8870983af660798dc1b529e1fd6f1cefd94e45135a32e58bd70edd694540f33", + "sha256:ac0adfdb3a21dc2a24ed728b61e72440d297d0fd3a577389df566651fcd08f97", + "sha256:b395121e9bbe8d02a750886f108d540abe66075e61e22f7353d9acb0b81be0f0", + "sha256:b9505a0c8579899057cbefd4ec34d865ab99852baf1ff33a9481eb3924e2da0b", + "sha256:c0a5b1c22c82831f56f2f7ad9bbe4948879762fe0d59833a4a71f16e5fa0f682", + "sha256:c3967dcc1cd2ea61b08b0b276659242cbce5caca39e7cbc02408222fb9e6ff39", + "sha256:c6f4c2027689093775fd58ca2388d58789009116844432d920e9147f91acbe64", + "sha256:c9d86401550b09a55410f32ceb5fe7efcd998bd2dad9e82521713cb148a4a15f", + "sha256:cd468ec62257bb4544989402b19d795d2305eccb06cde5da0eb739b63dc04665", + "sha256:cfcfb73aed40f550a57ea904629bdaf2e562c68fa1164fa4588e752af6efdc3f", + "sha256:d0dd943282231480aad5f50f89bdf26690c995e8ff555f26d8a5b9887b559bcc", + "sha256:d3c59a06c2c28a81a026ff11fbf012081ea34fb9b7052f2ed0366e14896f0a1d", + "sha256:d45b75b0f3fd8d99f62eb7908cfa6d727b7ed190737dec7fe46d993da550b81a", + "sha256:d46d5069e2eeda111d6f71970e341f4bd9aeeee92074e649ae263b834286ecc0", + "sha256:d58ec349e0c2c0bc6669bf2cd4982d2f93bf067860d23a0ea1fe677b0f0b1e09", + "sha256:db1b3ccb93488328c74e97ff888604a8b95ae4f35f4f56677ca57a4fc3a4220b", + "sha256:dd65695a8df1233309b701dec2539cc4b11e97d4fcc0f4185b4a12ce54db0491", + "sha256:f9482c2ed414781c0af0b35d9d575226da6b728bd1a720668fa05837184965b7", + "sha256:f9671e7282d8c6fcabc32c0fb8d7c0ea8894ae85cee89c9aadc2d7129e1a9954", + "sha256:fad7a051e07f64e297e6e8399b4d6a3bdcad3d7297409e9a06ef8cbccff4f501", + "sha256:ffb08f2a1e59d38c7b8b9ac8083c9c8b9875f0955b1e9b9b9a965607a51f8e54" + ], + "markers": "python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", + "version": "==3.1.0" + }, "httplib2": { "hashes": [ - "sha256:749c32603f9bf16c1277f59531d502e8f1c2ca19901ae653b49c4ed698f0820e", - "sha256:e0d428dad43c72dbce7d163b7753ffc7a39c097e6788ef10f4198db69b92f08e" + "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc", + "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81" ], "index": "pypi", - "version": "==0.19.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.22.0" + }, + "numpy": { + "hashes": [ + "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f", + "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61", + "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7", + "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400", + "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef", + "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2", + "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d", + "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc", + "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835", + "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706", + "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5", + "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4", + "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6", + "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463", + "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a", + "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f", + "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e", + "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e", + "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694", + "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8", + "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64", + "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d", + "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc", + "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254", + "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2", + "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1", + "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810", + "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9" + ], + "markers": "python_version < '3.10'", + "version": "==1.24.4" + }, + "pandas": { + "hashes": [ + "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682", + "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc", + "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b", + "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089", + "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5", + "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26", + "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210", + "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b", + "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641", + "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd", + "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78", + "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b", + "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e", + "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061", + "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0", + "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e", + "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8", + "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d", + "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0", + "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c", + "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183", + "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df", + "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8", + "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f", + "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==2.0.3" }, "psycopg2-binary": { "hashes": [ - "sha256:040234f8a4a8dfd692662a8308d78f63f31a97e1c42d2480e5e6810c48966a29", - "sha256:086f7e89ec85a6704db51f68f0dcae432eff9300809723a6e8782c41c2f48e03", - "sha256:18ca813fdb17bc1db73fe61b196b05dd1ca2165b884dd5ec5568877cabf9b039", - "sha256:19dc39616850342a2a6db70559af55b22955f86667b5f652f40c0e99253d9881", - "sha256:2166e770cb98f02ed5ee2b0b569d40db26788e0bf2ec3ae1a0d864ea6f1d8309", - "sha256:3a2522b1d9178575acee4adf8fd9f979f9c0449b00b4164bb63c3475ea6528ed", - "sha256:3aa773580f85a28ffdf6f862e59cb5a3cc7ef6885121f2de3fca8d6ada4dbf3b", - "sha256:3b5deaa3ee7180585a296af33e14c9b18c218d148e735c7accf78130765a47e3", - "sha256:407af6d7e46593415f216c7f56ba087a9a42bd6dc2ecb86028760aa45b802bd7", - "sha256:4c3c09fb674401f630626310bcaf6cd6285daf0d5e4c26d6e55ca26a2734e39b", - "sha256:4c6717962247445b4f9e21c962ea61d2e884fc17df5ddf5e35863b016f8a1f03", - "sha256:50446fae5681fc99f87e505d4e77c9407e683ab60c555ec302f9ac9bffa61103", - "sha256:5057669b6a66aa9ca118a2a860159f0ee3acf837eda937bdd2a64f3431361a2d", - "sha256:5dd90c5438b4f935c9d01fcbad3620253da89d19c1f5fca9158646407ed7df35", - "sha256:659c815b5b8e2a55193ede2795c1e2349b8011497310bb936da7d4745652823b", - "sha256:69b13fdf12878b10dc6003acc8d0abf3ad93e79813fd5f3812497c1c9fb9be49", - "sha256:7a1cb80e35e1ccea3e11a48afe65d38744a0e0bde88795cc56a4d05b6e4f9d70", - "sha256:7e6e3c52e6732c219c07bd97fff6c088f8df4dae3b79752ee3a817e6f32e177e", - "sha256:7f42a8490c4fe854325504ce7a6e4796b207960dabb2cbafe3c3959cb00d1d7e", - "sha256:84156313f258eafff716b2961644a4483a9be44a5d43551d554844d15d4d224e", - "sha256:8578d6b8192e4c805e85f187bc530d0f52ba86c39172e61cd51f68fddd648103", - "sha256:890167d5091279a27e2505ff0e1fb273f8c48c41d35c5b92adbf4af80e6b2ed6", - "sha256:98e10634792ac0e9e7a92a76b4991b44c2325d3e7798270a808407355e7bb0a1", - "sha256:9aadff9032e967865f9778485571e93908d27dab21d0fdfdec0ca779bb6f8ad9", - "sha256:9f24f383a298a0c0f9b3113b982e21751a8ecde6615494a3f1470eb4a9d70e9e", - "sha256:a73021b44813b5c84eda4a3af5826dd72356a900bac9bd9dd1f0f81ee1c22c2f", - "sha256:afd96845e12638d2c44d213d4810a08f4dc4a563f9a98204b7428e567014b1cd", - "sha256:b73ddf033d8cd4cc9dfed6324b1ad2a89ba52c410ef6877998422fcb9c23e3a8", - "sha256:b8f490f5fad1767a1331df1259763b3bad7d7af12a75b950c2843ba319b2415f", - "sha256:dbc5cd56fff1a6152ca59445178652756f4e509f672e49ccdf3d79c1043113a4", - "sha256:eac8a3499754790187bb00574ab980df13e754777d346f85e0ff6df929bcd964", - "sha256:eaed1c65f461a959284649e37b5051224f4db6ebdc84e40b5e65f2986f101a08" + "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9", + "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77", + "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e", + "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84", + "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3", + "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2", + "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67", + "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876", + "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152", + "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f", + "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a", + "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6", + "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503", + "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f", + "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493", + "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996", + "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f", + "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e", + "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59", + "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94", + "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7", + "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682", + "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420", + "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae", + "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291", + "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe", + "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980", + "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93", + "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692", + "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119", + "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716", + "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472", + "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b", + "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2", + "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc", + "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c", + "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5", + "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab", + "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984", + "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9", + "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf", + "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0", + "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f", + "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212", + "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb", + "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be", + "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90", + "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041", + "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7", + "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860", + "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d", + "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245", + "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27", + "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417", + "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359", + "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202", + "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0", + "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7", + "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba", + "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1", + "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd", + "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07", + "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98", + "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55", + "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d", + "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972", + "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f", + "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e", + "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26", + "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957", + "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53", + "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52" ], "index": "pypi", - "version": "==2.8.4" + "markers": "python_version >= '3.7'", + "version": "==2.9.9" }, "pyparsing": { "hashes": [ - "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", - "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", + "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032" + ], + "markers": "python_version >= '3.1'", + "version": "==3.1.4" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.9.0.post0" + }, + "pytz": { + "hashes": [ + "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a", + "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725" + ], + "version": "==2024.2" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "sqlalchemy": { + "hashes": [ + "sha256:02d2ecb9508f16ab9c5af466dfe5a88e26adf2e1a8d1c56eb616396ccae2c186", + "sha256:0b76bbb1cbae618d10679be8966f6d66c94f301cfc15cb49e2f2382563fb6efb", + "sha256:0de620f978ca273ce027769dc8db7e6ee72631796187adc8471b3c76091b809e", + "sha256:1183599e25fa38a1a322294b949da02b4f0da13dbc2688ef9dbe746df573f8a6", + "sha256:12bc0141b245918b80d9d17eca94663dbd3f5266ac77a0be60750f36102bbb0f", + "sha256:1390ca2d301a2708fd4425c6d75528d22f26b8f5cbc9faba1ddca136671432bc", + "sha256:13e91d6892b5fcb94a36ba061fb7a1f03d0185ed9d8a77c84ba389e5bb05e936", + "sha256:14b3f4783275339170984cadda66e3ec011cce87b405968dc8d51cf0f9997b0d", + "sha256:1576fba3616f79496e2f067262200dbf4aab1bb727cd7e4e006076686413c80c", + "sha256:1990d5a6a5dc358a0894c8ca02043fb9a5ad9538422001fb2826e91c50f1d539", + "sha256:1d83cd1cc03c22d922ec94d0d5f7b7c96b1332f5e122e81b1a61fb22da77879a", + "sha256:1e8c1b9ecaf9f2590337d5622189aeb2f0dbc54ba0232fa0856cf390957584a9", + "sha256:26e78444bc77d089e62874dc74df05a5c71f01ac598010a327881a48408d0064", + "sha256:2b37931eac4b837c45e2522066bda221ac6d80e78922fb77c75eb12e4dbcdee5", + "sha256:3112de9e11ff1957148c6de1df2bc5cc1440ee36783412e5eedc6f53638a577d", + "sha256:394b0135900b62dbf63e4809cdc8ac923182af2816d06ea61cd6763943c2cc05", + "sha256:3f01c2629a7d6b30d8afe0326b8c649b74825a0e1ebdcb01e8ffd1c920deb07d", + "sha256:41cffc63c7c83dfc30c4cab5b4308ba74440a9633c4509c51a0c52431fb0f8ab", + "sha256:4470fbed088c35dc20b78a39aaf4ae54fe81790c783b3264872a0224f437c31a", + "sha256:5ed3576675c187e3baa80b02c4c9d0edfab78eff4e89dd9da736b921333a2432", + "sha256:6b24364150738ce488333b3fb48bfa14c189a66de41cd632796fbcacb26b4585", + "sha256:6da60fb24577f989535b8fc8b2ddc4212204aaf02e53c4c7ac94ac364150ed08", + "sha256:76c2ba7b5a09863d0a8166fbc753af96d561818c572dbaf697c52095938e7be4", + "sha256:954816850777ac234a4e32b8c88ac1f7847088a6e90cfb8f0e127a1bf3feddff", + "sha256:9c24dd161c06992ed16c5e528a75878edbaeced5660c3db88c820f1f0d3fe1f4", + "sha256:a01bc25eb7a5688656c8770f931d5cb4a44c7de1b3cec69b84cc9745d1e4cc10", + "sha256:a19f816f4702d7b1951d7576026c7124b9bfb64a9543e571774cf517b7a50b29", + "sha256:a41611835010ed4ea4c7aed1da5b58aac78ee7e70932a91ed2705a7b38e40f52", + "sha256:a49730afb716f3f675755afec109895cab95bc9875db7ffe2e42c1b1c6279482", + "sha256:a86b0e4be775902a5496af4fb1b60d8a2a457d78f531458d294360b8637bb014", + "sha256:a8a72259a1652f192c68377be7011eac3c463e9892ef2948828c7d58e4829988", + "sha256:af00236fe21c4d4f4c227b6ccc19b44c594160cc3ff28d104cdce85855369277", + "sha256:b05e0626ec1c391432eabb47a8abd3bf199fb74bfde7cc44a26d2b1b352c2c6e", + "sha256:b5933c45d11cbd9694b1540aa9076816cc7406964c7b16a380fd84d3a5fe3241", + "sha256:b5e0d47d619c739bdc636bbe007da4519fc953393304a5943e0b5aec96c9877c", + "sha256:b67589f7955924865344e6eacfdcf70675e64f36800a576aa5e961f0008cde2a", + "sha256:c5a2530400a6e7e68fd1552a55515de6a4559122e495f73554a51cedafc11669", + "sha256:cafe0ba3a96d0845121433cffa2b9232844a2609fce694fcc02f3f31214ece28", + "sha256:cdb2886c0be2c6c54d0651d5a61c29ef347e8eec81fd83afebbf7b59b80b7393", + "sha256:d0cf7076c8578b3de4e43a046cc7a1af8466e1c3f5e64167189fe8958a4f9c02", + "sha256:f1e1b92ee4ee9ffc68624ace218b89ca5ca667607ccee4541a90cc44999b9aea", + "sha256:f941aaf15f47f316123e1933f9ea91a6efda73a161a6ab6046d1cde37be62c88", + "sha256:fb59a11689ff3c58e7652260127f9e34f7f45478a2f3ef831ab6db7bcd72108f", + "sha256:fc9ffd9a38e21fad3e8c5a88926d57f94a32546e937e0be46142b2702003eba7" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.4.54" + }, + "tzdata": { + "hashes": [ + "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd", + "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252" ], - "version": "==2.4.7" + "markers": "python_version >= '2'", + "version": "==2024.1" } }, "develop": {} diff --git a/operations/auditing/README.md b/operations/auditing/README.md index d32f0412..6a20db02 100644 --- a/operations/auditing/README.md +++ b/operations/auditing/README.md @@ -10,7 +10,7 @@ The script *createAuditReport.py* in this folder generates an auditing report ab ## Prerequisites * The process [Copy Looker SQL query Logs](../../maintenance/copySqlQuery) is in place * The machine running the script must have - * Python 2.7 + * Python 3.8 * [pipenv](https://github.com/pypa/pipenv) * access to Redshift * access to Looker API endpoint @@ -30,15 +30,13 @@ Optionally, set following environment variables if you don't want to supply the * PGHOST - Redshift host * PGUSER - Redshift user name * PGPASSWORD - Redshift user password -* lookerUrlPrefix - Looker url prefix such as https://looker.local:19999/api/3.1 +* lookerUrlPrefix - Looker url prefix such as https://looker.local:19999/api/4.0 ## Usage ``` $ pipenv run python createAuditReport.py -h -usage: createAuditReport.py [-h] [-s LOOKERCLIENTSECRET] [-H PGHOST] - [-u PGUSER] [-p PGPASSWORD] [-l LOOKERURLPREFIX] - siteHost lookerClientId +usage: createAuditReport.py [-h] [-s LOOKERCLIENTSECRET] [-H PGHOST] [-u PGUSER] [-p PGPASSWORD] [-l LOOKERURLPREFIX] [-f FILE] siteHost lookerClientId Generate auditing report for a given site host. @@ -57,21 +55,20 @@ optional arguments: -p PGPASSWORD, --pgPassword PGPASSWORD Redshift user password overriding env var PGPASSWORD -l LOOKERURLPREFIX, --lookerUrlPrefix LOOKERURLPREFIX - Looker url prefix such as - https://looker.local:19999/api/3.1 overriding env var - lookerUrlPrefix + Looker url prefix such as https://looker.local:19999/api/3.1 overriding env var lookerUrlPrefix + -f FILE, --file FILE Write results to a csv file at the specified path ``` If any required information is neither supplied in env var nor cmd arg, you will be prompted to enter it. Examples ``` -$ pipenv run python createAuditReport.py foo.com myLookerClientId -s myLookerClientSecret > report.csv +$ pipenv run python createAuditReport.py foo.com myLookerClientId -s myLookerClientSecret -f report.csv Enter your looker client secret: ``` Generate report for site *foo.com* into file *report.csv* if PGHOST, PGUSER, PGPASSWORD and lookerUrlPrefix have been supplied via environment variables. ``` -$ pipenv run python createAuditReport.py -H redshift.host -u reshift_user -p reshift_user_password -l https://looker.local:19999/api/3.1 -s myLookerClientSecret foo.com myLookerClientId > report.csv +$ pipenv run python createAuditReport.py -H redshift.host -u reshift_user -p reshift_user_password -l https://looker.local:19999/api/4.0 -s myLookerClientSecret www.foo.com myLookerClientId -f report.csv ``` Generate report for site *foo.com* into file *report.csv*, supplying all required information via command arguments. diff --git a/operations/auditing/createAuditReport.py b/operations/auditing/createAuditReport.py index 20d7d83b..4cbefcfb 100644 --- a/operations/auditing/createAuditReport.py +++ b/operations/auditing/createAuditReport.py @@ -11,11 +11,13 @@ import json import argparse import getpass +import pandas as pd +from sqlalchemy import create_engine # parse cmd arguments parser = argparse.ArgumentParser( description='Generate auditing report for a given site host.') -parser.add_argument('siteHost', help='site host such as my-site.gov.bc.ca') +parser.add_argument('siteHost', help='site host such as www.my-site.gov.bc.ca') parser.add_argument('lookerClientId', help='your looker client id') parser.add_argument('-s', '--lookerClientSecret', @@ -33,42 +35,48 @@ parser.add_argument('-l', '--lookerUrlPrefix', help='Looker url prefix such as https://looker.local:19999' - '/api/3.1 overriding env var lookerUrlPrefix') + '/api/4.0 overriding env var lookerUrlPrefix') +parser.add_argument('-f', + '--file', + help='Write results to a csv file at the specified path') args = parser.parse_args() +# set var from cmd args or envvars lookerClientId = args.lookerClientId pgHost = args.pgHost or os.getenv('PGHOST') pgUser = args.pgUser or os.getenv('PGUSER') pgPassword = args.pgPassword or os.getenv('PGPASSWORD') lookerUrlPrefix = args.lookerUrlPrefix or os.getenv('lookerUrlPrefix') lookerClientSecret = args.lookerClientSecret +file = args.file # prompt user to supply missing mandatory information if pgHost is None: - sys.stderr.write('Enter Redshift host name: ') - pgHost = sys.stdin.readline().strip() + pgHost = input('Enter Redshift host name: ') if pgUser is None: - sys.stderr.write('Enter Redshift user name: ') - pgUser = sys.stdin.readline().strip() + pgUser = input('Enter Redshift user name: ') if pgPassword is None: - sys.stderr.write('Enter Redshift user password: ') - pgPassword = sys.stdin.readline().strip() + pgPassword = getpass.getpass('Enter Redshift user password: ') if lookerUrlPrefix is None: - sys.stderr.write('Enter Looker Url prefix: ') - lookerUrlPrefix = sys.stdin.readline().strip() + lookerUrlPrefix = input('Enter Looker Url prefix: ') if lookerClientSecret is None: lookerClientSecret = getpass.getpass('Enter your looker client secret: ') # database connection string -conn_string = "dbname='snowplow' host='" + pgHost + \ - "' port='5439' user='" + pgUser + "' password=" + pgPassword +connection_string = "postgresql+psycopg2://{}:{}@{}:{}/{}".format( + pgUser, + pgPassword, + pgHost, + '5439', + 'snowplow' +) +engine = create_engine(connection_string) -# query database +# create empty dict for api user data lookerUserIdNameMap = {} -with psycopg2.connect(conn_string) as conn: - with conn.cursor() as curs: - qryString = """ + +qryString = """ select convert_timezone('America/Vancouver', starttime), looker_user_id, @@ -76,60 +84,94 @@ from admin.V_STL_QUERY_HISTORY where - sqlquery ilike '-- Looker Query Context%page_urlhost%' + sqlquery ilike '-- Looker Query Context%%page_urlhost%%' -- exclude known admin users and looker_user_id not in ('13','128','8','513','35','746','742','6','16') and ( - sqlquery ilike '%page_urlhost)) = ''{0}''%' + sqlquery ilike '%%page_urlhost)) = ''{0}''%%' or - sqlquery ilike '%page_urlhost = ''{0}''%' + sqlquery ilike '%%page_urlhost = ''{0}''%%' or - position('page_urlhost LIKE ''%''' in sqlquery) > 0 + position('page_urlhost LIKE ''%%''' in sqlquery) > 0 ) order by starttime desc; """.format(args.siteHost) # nosec - curs.execute(qryString) - for rec in curs: - lookerUserIdNameMap[rec[1]] = {} - # query looker users - h = httplib2.Http( - ca_certs=os.path.dirname(os.path.realpath(__file__)) + '/ca.crt') - resp, content = h.request( - lookerUrlPrefix + '/login', - "POST", - body='client_id=' + lookerClientId + '&client_secret=' + - lookerClientSecret, - headers={'content-type': 'application/' - 'x-www-form-urlencoded'}) - jsonRes = json.loads(content) - accessToken = jsonRes['access_token'] - userSrchStr = lookerUrlPrefix+'/users/search?id=' + \ - ','.join(lookerUserIdNameMap.keys()) - resp, content = h.request( - userSrchStr, - "GET", - headers={'authorization': 'token ' + accessToken}) - users = json.loads(content) - for usr in users: - lookerUserIdNameMap[str( - usr['id'])]["displayName"] = usr['display_name'] - if usr['credentials_embed'] is not None and len( - usr['credentials_embed']) > 0: - lookerUserIdNameMap[str( - usr['id'])]["embedUserId"] = \ - usr['credentials_embed'][0]['external_user_id'] - print('"Date & Time","Looker User Id",' - '"User Display Name","User Embed Id","Query Text"') - curs.scroll(0, mode='absolute') - for rec in curs: - displayNm = lookerUserIdNameMap[rec[1]].get("displayName", - '').replace('"', '""') - userEmbedId = lookerUserIdNameMap[rec[1]].get("embedUserId", - '').replace( - '"', '""') - queryTxt = rec[2] - if queryTxt is not None: - queryTxt = queryTxt.replace('"', '""') - print('{0},{1},"{2}","{3}","{4}"'.format(rec[0], rec[1], displayNm, - userEmbedId, queryTxt)) + +# execute the query and store in a dataframe +dfQuery = pd.read_sql(qryString, engine).fillna('') + +# grab the unique user ids from the query +queryUserIdList = dfQuery['looker_user_id'].unique() +for id in queryUserIdList: + lookerUserIdNameMap[id] = {} + +# api login to looker +h = httplib2.Http( + ca_certs=os.path.dirname(os.path.realpath(__file__)) + '/ca.crt') +resp, content = h.request( + lookerUrlPrefix + '/login', + "POST", + body='client_id=' + lookerClientId + '&client_secret=' + + lookerClientSecret, + headers={'content-type': 'application/' + 'x-www-form-urlencoded'}) +jsonRes = json.loads(content) +accessToken = jsonRes['access_token'] +userSrchStr = lookerUrlPrefix+'/users/search?id=' + \ + ','.join(lookerUserIdNameMap.keys()) + +# api call for looker users +resp, content = h.request( + userSrchStr, + "GET", + headers={'authorization': 'token ' + accessToken}) +users = json.loads(content) + +# add looker user data to dict if user id in query +for usr in users: + lookerUserIdNameMap[str( + usr['id'])]["displayName"] = usr['display_name'] + if usr['credentials_embed'] is not None and len( + usr['credentials_embed']) > 0: + lookerUserIdNameMap[str( + usr['id'])]["embedUserId"] = \ + usr['credentials_embed'][0]['external_user_id'] + else: + # guarentees that embed id is added to the dict even if + # missing from all users in query results + lookerUserIdNameMap[str( + usr['id'])]["embedUserId"] = None +# convert name map dict to dataframe +dfLookerUserIdNameMap = pd.DataFrame.from_dict(lookerUserIdNameMap, orient='index').fillna('') +dfLookerUserIdNameMap = dfLookerUserIdNameMap.reset_index() +dfLookerUserIdNameMap = dfLookerUserIdNameMap.rename(columns={"index": "looker_user_id"}) + +# merge query results with name map +dfMerged = pd.merge(dfQuery, dfLookerUserIdNameMap, on='looker_user_id') + +# Cleaning up the header names +dfMerged = dfMerged.rename(columns={"convert_timezone": "Date & Time", + "looker_user_id": "Looker User Id", + "displayName": "User Display Name", + "embedUserId": "User Embed Id", + "sqlquery": "Query Text"}) + +# setting column datatypes +dfMerged['Date & Time'] = dfMerged['Date & Time'].astype('datetime64[ns]') +dfMerged['Looker User Id'] = dfMerged['Looker User Id'].astype('int') +dfMerged['User Display Name'] = dfMerged['User Display Name'].astype('str') +dfMerged['User Embed Id'] = dfMerged['User Embed Id'].astype('str') +dfMerged['Query Text'] = dfMerged['Query Text'].astype('str') + +# reorder columns +dfMerged = dfMerged.iloc[:,[0,1,3,4,2]] + +# replace NaN with blanks +dfMerged = dfMerged.fillna('') + +# print results to console# if output is not set print results to console +if file is None: + print(dfMerged.to_csv(index=False, quoting=2)) +else: + dfMerged.to_csv(path_or_buf=file, index=False, quoting=2)