Environment seperation cleanup, subscription API (#53)

* Env seperation cleanup, subscription API

* print trim

* trimming debug

* mocker linting, 3.9 cleanup, resorted athena permissions

* manual lint run and correction

* documentation branch leak plugging

* black targeting

.gitignore

@@ -1,8 +1,10 @@
 .aws-sam
+.python-version
 samconfig.toml
 data/
 auth.json
 metadata.json
+*.png
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.pre-commit-config.yaml

@@ -9,4 +9,4 @@ repos:
         # supported by your project here, or alternatively use
         # pre-commit's default_language_version, see
         # https://pre-commit.com/#top_level-default_language_version
-        language_version: python3.11
+        language_version: python3.9

events/event-get-chart-data.json

@@ -0,0 +1,125 @@
+{
+    "resource": "/chart_data/{subscription_name}",
+    "path": "/chart_data/covid__encounter",
+    "httpMethod": "GET",
+    "headers": {
+        "Accept": "*/*",
+        "Accept-Encoding": "gzip, deflate, br",
+        "CloudFront-Forwarded-Proto": "https",
+        "CloudFront-Is-Desktop-Viewer": "true",
+        "CloudFront-Is-Mobile-Viewer": "false",
+        "CloudFront-Is-SmartTV-Viewer": "false",
+        "CloudFront-Is-Tablet-Viewer": "false",
+        "CloudFront-Viewer-ASN": "40127",
+        "CloudFront-Viewer-Country": "US",
+        "Host": "effmuaxft2.execute-api.us-east-1.amazonaws.com",
+        "Postman-Token": "10f731c3-bb8d-4720-971c-7fd5e7cf0065",
+        "User-Agent": "PostmanRuntime/7.31.3",
+        "Via": "1.1 3ae9464b3a12f9a00e97e3c81ee98466.cloudfront.net (CloudFront)",
+        "X-Amz-Cf-Id": "JWyqc8QN24nqpUdGuej_gc35vpExxV017XJD8EXxM8x1OupUzCGYbg==",
+        "X-Amzn-Trace-Id": "Root=1-641c7f2c-5c612f3e389094297688efa0",
+        "X-Forwarded-For": "134.174.21.156, 70.132.21.75",
+        "X-Forwarded-Port": "443",
+        "X-Forwarded-Proto": "https"
+    },
+    "multiValueHeaders": {
+        "Accept": [
+            "*/*"
+        ],
+        "Accept-Encoding": [
+            "gzip, deflate, br"
+        ],
+        "CloudFront-Forwarded-Proto": [
+            "https"
+        ],
+        "CloudFront-Is-Desktop-Viewer": [
+            "true"
+        ],
+        "CloudFront-Is-Mobile-Viewer": [
+            "false"
+        ],
+        "CloudFront-Is-SmartTV-Viewer": [
+            "false"
+        ],
+        "CloudFront-Is-Tablet-Viewer": [
+            "false"
+        ],
+        "CloudFront-Viewer-ASN": [
+            "40127"
+        ],
+        "CloudFront-Viewer-Country": [
+            "US"
+        ],
+        "Host": [
+            "effmuaxft2.execute-api.us-east-1.amazonaws.com"
+        ],
+        "Postman-Token": [
+            "10f731c3-bb8d-4720-971c-7fd5e7cf0065"
+        ],
+        "User-Agent": [
+            "PostmanRuntime/7.31.3"
+        ],
+        "Via": [
+            "1.1 3ae9464b3a12f9a00e97e3c81ee98466.cloudfront.net (CloudFront)"
+        ],
+        "X-Amz-Cf-Id": [
+            "JWyqc8QN24nqpUdGuej_gc35vpExxV017XJD8EXxM8x1OupUzCGYbg=="
+        ],
+        "X-Amzn-Trace-Id": [
+            "Root=1-641c7f2c-5c612f3e389094297688efa0"
+        ],
+        "X-Forwarded-For": [
+            "134.174.21.156, 70.132.21.75"
+        ],
+        "X-Forwarded-Port": [
+            "443"
+        ],
+        "X-Forwarded-Proto": [
+            "https"
+        ]
+    },
+    "queryStringParameters": {
+        "column": "gender"
+    },
+    "multiValueQueryStringParameters": {
+        "column": [
+            "gender"
+        ]
+    },
+    "pathParameters": {
+        "subscription_name": "covid__encounter"
+    },
+    "stageVariables": "None",
+    "requestContext": {
+        "resourceId": "ta1ris",
+        "resourcePath": "/chart_data/{subscription_name}",
+        "httpMethod": "GET",
+        "extendedRequestId": "CPjPEHgQoAMFZ8A=",
+        "requestTime": "23/Mar/2023:16:32:44 +0000",
+        "path": "/dev/chart_data/covid__encounter",
+        "accountId": "316333106222",
+        "protocol": "HTTP/1.1",
+        "stage": "dev",
+        "domainPrefix": "effmuaxft2",
+        "requestTimeEpoch": 1679589164970,
+        "requestId": "ec3e7961-c978-45b5-af6b-46b70bcd9d7c",
+        "identity": {
+            "cognitoIdentityPoolId": "None",
+            "accountId": "None",
+            "cognitoIdentityId": "None",
+            "caller": "None",
+            "sourceIp": "134.174.21.156",
+            "principalOrgId": "None",
+            "accessKey": "None",
+            "cognitoAuthenticationType": "None",
+            "cognitoAuthenticationProvider": "None",
+            "userArn": "None",
+            "userAgent": "PostmanRuntime/7.31.3",
+            "user": "None"
+        },
+        "domainName": "effmuaxft2.execute-api.us-east-1.amazonaws.com",
+        "apiId": "effmuaxft2"
+    },
+    "body": "None",
+    "isBase64Encoded": false
+}

pyproject.toml

@@ -0,0 +1,50 @@
+[project]
+name = "aggregator"
+version = "0.1.0"
+# This project is designed to run on the AWS serverless application framework (SAM).
+# The project dependencies are handled via AWS layers. These are only required for
+# local development.
+dependencies= [
+    "awswrangler",
+    "boto3",
+    "pandas"
+]
+authors = [
+  { name="Matt Garber", email="[email protected]" },
+]
+description = "Aggregates data from distributed hopsitals for display in Cumulus dashboard"
+readme = "README.md"
+license = { text="Apache License 2.0" }
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/smart-on-fhir/cumulus-aggregator"
+
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = [
+    "setuptools ~=63.2.0",
+    "wheel ~=0.37.1",
+]
+
+[project.optional-dependencies]
+test = [
+    # TODO: We are investigating some new features around athena mocks, 
+    # which currently only exist in 4.1.5 and are broken in 4.1.6dev. Swing back
+    # around on this as the functionality matures.
+    "moto[s3,athena] == 4.1.5",
+    "pytest",
+    "pytest-mock"
+]
+dev = [
+    "bandit",
+    "black",
+    "pre-commit",
+    "pylint",
+    "pycodestyle"
+
+]

scripts/cumulus_upload_data.py

@@ -25,16 +25,18 @@ def upload_file(args):
         try:
             api_client = boto3.client("apigateway")
             res = api_client.get_rest_apis()
-            site_api_dict = list(
+            api_dict = list(
                 filter(
                     lambda x: "cumulus-aggregator-dev"
                     in x["tags"]["aws:cloudformation:stack-name"],
                     res["items"],
                 )
             )
-            api_id = site_api_dict[0]["id"]
-            url = f"https://{api_id}.execute-api.us-east-1.amazonaws.com/dev/"
-
+            for api in api_dict:
+                if api["name"] == "CumulusAggregatorSiteAPI":
+                    url = (
+                        f"https://{api['id']}.execute-api.us-east-1.amazonaws.com/dev/"
+                    )
         except:
             print("No response recieved from AWS API gateway.")
             exit(1)