Consolidate unit test S3 mocking (#52)

* Consolidate unit test S3 mocking

* isolated conftest scope

* remove unintended file

* disabled autouse, reset test structure

* typo

* re-remove WIP file

README.md

@@ -12,7 +12,7 @@ The aggregator aims to provide a serverless implementation that accomplish the f
 * AWS SAM CLI - [Install the AWS SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html)
 * Docker - [Install Docker community edition](https://hub.docker.com/search/?type=edition&offering=community) - for local development
 
-## Working with the SAM CLI
+## Working with SAM CLI
 
 ### Deployment Prequisites
 
@@ -84,7 +84,7 @@ If for some reason you don't want to use the samconfig.toml, you could instead d
 Assuming you have done the above configuration, you can deploy the aggregator with the following command:
 
 ```bash
-sam deploy --config-env [stagename]
+sam build && sam deploy --config-env [stagename]
 ```
 
 This will show you a list of changes to your environment and ask you to confirm. Responding to the prompt with 'Y' will kick off the deployment.

scripts/cumulus_upload_data.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # basic utility for debugging merge behavior
 
 import argparse
@@ -26,13 +27,14 @@ def upload_file(args):
             res = api_client.get_rest_apis()
             site_api_dict = list(
                 filter(
-                    lambda x: x["tags"]["aws:cloudformation:logical-id"]
-                    == "SiteApiGateway",
+                    lambda x: "cumulus-aggregator-dev"
+                    in x["tags"]["aws:cloudformation:stack-name"],
                     res["items"],
                 )
             )
             api_id = site_api_dict[0]["id"]
             url = f"https://{api_id}.execute-api.us-east-1.amazonaws.com/dev/"
+
         except:
             print("No response recieved from AWS API gateway.")
             exit(1)
@@ -83,7 +85,7 @@ def upload_file(args):
     )
     parser.add_argument("-f", "--file", help="The data file to upload")
     parser.add_argument("-u", "--user", help="the name of the site uploading data")
-    parser.add_argument("-p", "--pass", help="the secret assigned to a site")
+    parser.add_argument("-a", "--auth", help="the secret assigned to a site")
     parser.add_argument("-n", "--studyname", help="the name of the data's study")
     parser.add_argument(
         "-s", "--subscription", help="the subscription name within the study"
@@ -104,7 +106,7 @@ def upload_file(args):
         args_dict["user"] = "general"
         args_dict[
             "file"
-        ] = f"{os.path.realpath(os.path.dirname(__file__))}/cube_simple_example.csv"
+        ] = f"{str(Path(__file__).resolve().parents[1])}/tests/test_data/cube_simple_example.parquet"
         args_dict["auth"] = "secretval"
         args_dict["studyname"] = "covid"
         args_dict["subscription"] = "encounter"

src/handlers/site_upload/powerset_merge.py

@@ -194,13 +194,13 @@ def merge_powersets(
     write_metadata(s3_client, s3_bucket_name, metadata)
     csv_aggregate_path = (
         f"s3://{s3_bucket_name}/{BucketPath.CSVAGGREGATE.value}/"
-        f"{study}/{study}_{subscription}/{study}_{subscription}_aggregate.csv"
+        f"{study}/{study}__{subscription}/{study}__{subscription}__aggregate.csv"
     )
     df = df.apply(lambda x: x.strip() if isinstance(x, str) else x).replace('""', nan)
     awswrangler.s3.to_csv(df, csv_aggregate_path, index=False, quoting=csv.QUOTE_NONE)
     aggregate_path = (
         f"s3://{s3_bucket_name}/{BucketPath.AGGREGATE.value}/"
-        f"{study}/{study}_{subscription}/{study}_{subscription}_aggregate.parquet"
+        f"{study}/{study}__{subscription}/{study}__{subscription}__aggregate.parquet"
     )
     # Note: the to_parquet function is noted in the docs as potentially mutating the
     # dataframe it's called on, so this should always be the last action applied

tests/conftest.py

@@ -0,0 +1,52 @@
+import boto3
+import json
+import os
+
+import pytest
+from moto import mock_s3, mock_athena
+from unittest import mock
+
+from src.handlers.shared.enums import BucketPath
+from src.handlers.shared.functions import read_metadata, write_metadata
+from tests.utils import get_mock_metadata, get_mock_auth, TEST_BUCKET, ITEM_COUNT
+
+
+def _init_mock_data(s3_client, bucket_name, site, study, subscription):
+    s3_client.upload_file(
+        "./tests/test_data/cube_simple_example.parquet",
+        bucket_name,
+        f"{BucketPath.AGGREGATE.value}/{site}/{study}/"
+        f"{site}__{subscription}/{site}__{subscription}__aggregate.parquet",
+    )
+    s3_client.upload_file(
+        "./tests/test_data/cube_simple_example.csv",
+        bucket_name,
+        f"{BucketPath.CSVAGGREGATE.value}/{site}/{study}/"
+        f"{site}__{subscription}/{site}__{subscription}__aggregate.csv",
+    )
+
+
+@pytest.fixture
+def mock_bucket():
+    """Mock for testing S3 usage. Should reset before each individual test."""
+    s3 = mock_s3()
+    s3.start()
+    s3_client = boto3.client("s3", region_name="us-east-1")
+    s3_client.create_bucket(Bucket=TEST_BUCKET)
+    aggregate_params = [
+        ["general_hospital", "covid", "encounter"],
+        ["general_hospital", "lyme", "encounter"],
+        ["st_elsewhere", "covid", "encounter"],
+    ]
+    for param_list in aggregate_params:
+        _init_mock_data(s3_client, TEST_BUCKET, *param_list)
+    metadata = get_mock_metadata()
+    write_metadata(s3_client, TEST_BUCKET, metadata)
+    yield
+    s3.stop()
+
+
+def test_mock_bucket():
+    s3_client = boto3.client("s3", region_name="us-east-1")
+    item = s3_client.list_objects_v2(Bucket=TEST_BUCKET)
+    assert (len(item["Contents"])) == ITEM_COUNT

tests/dashboard/test_get_metadata.py

@@ -4,73 +4,15 @@
 
 import pytest
 from datetime import datetime, timezone
-from moto import mock_s3
 from unittest import mock
 
 from src.handlers.shared.enums import BucketPath
 from src.handlers.shared.functions import read_metadata, write_metadata
 from src.handlers.dashboard.get_metadata import metadata_handler
+from tests.utils import get_mock_metadata, TEST_BUCKET
 
 
-def get_mock_metadata():
-    return {
-        "general_hospital": {
-            "covid": {
-                "encounter": {
-                    "version": "1.0",
-                    "last_upload": "2023-02-24T15:03:34+00:00",
-                    "last_data_update": "2023-02-24T15:03:40.657583+00:00",
-                    "last_aggregation": "2023-02-24T15:08:07.504595+00:00",
-                    "last_error": None,
-                    "earliest_data": None,
-                    "latest_data": None,
-                    "deleted": None,
-                }
-            },
-            "lyme": {
-                "encounter": {
-                    "version": "1.0",
-                    "last_upload": "2023-02-24T15:43:57+00:00",
-                    "last_data_update": "2023-02-24T15:44:03.861574+00:00",
-                    "last_aggregation": "2023-02-24T15:44:03.861574+00:00",
-                    "last_error": None,
-                    "earliest_data": None,
-                    "latest_data": None,
-                    "deleted": None,
-                }
-            },
-        },
-        "st_elsewhere": {
-            "covid": {
-                "encounter": {
-                    "version": "1.0",
-                    "last_upload": "2023-02-24T15:08:06+00:00",
-                    "last_data_update": "2023-02-24T15:08:07.771080+00:00",
-                    "last_aggregation": "2023-02-24T15:08:07.771080+00:00",
-                    "last_error": None,
-                    "earliest_data": None,
-                    "latest_data": None,
-                    "deleted": None,
-                }
-            }
-        },
-    }
-
-
-@pytest.fixture(autouse=True)
-def mock_bucket():
-    mocks3 = mock_s3()
-    mocks3.start()
-    bucket_name = "cumulus-aggregator-site-counts"
-    metadata = get_mock_metadata()
-    s3_client = boto3.client("s3", region_name="us-east-1")
-    s3_client.create_bucket(Bucket=bucket_name)
-    write_metadata(s3_client, bucket_name, metadata)
-    yield
-    mocks3.stop()
-
-
-@mock.patch.dict(os.environ, {"BUCKET_NAME": "cumulus-aggregator-site-counts"})
+@mock.patch.dict(os.environ, {"BUCKET_NAME": TEST_BUCKET})
 @pytest.mark.parametrize(
     "params,status,expected",
     [
@@ -90,7 +32,7 @@ def mock_bucket():
         ({"site": "general_hospital", "study": "flu"}, 500, None),
     ],
 )
-def test_get_metadata(params, status, expected):
+def test_get_metadata(mock_bucket, params, status, expected):
     event = {"pathParameters": params}
 
     res = metadata_handler(event, {})

tests/site_upload/test_api_gateway_authorizer.py

@@ -1,31 +1,27 @@
+import json
 import pytest
 
 from contextlib import nullcontext as does_not_raise
 
 from src.handlers.site_upload.api_gateway_authorizer import lambda_handler
+from tests.utils import get_mock_auth
 
 
 @pytest.fixture
 def mocker_site_json(mocker):
-    mocked_site_json = mocker.mock_open(
-        read_data="""{
-            "testauth":{
-                "site":"general"
-            }
-        }"""
-    )
+    mocked_site_json = mocker.mock_open(read_data=json.dumps(get_mock_auth()))
     mocker.patch("builtins.open", mocked_site_json)
 
 
 @pytest.mark.parametrize(
     "auth,expects",
     [
-        ("Basic testauth", does_not_raise()),
-        ("Basic testauth2", pytest.raises(Exception)),
+        (f"Basic {list(get_mock_auth().keys())[0]}", does_not_raise()),
+        ("Basic other_auth", pytest.raises(Exception)),
         (None, pytest.raises(Exception)),
     ],
 )
-def test_validate_pw(auth, expects, mocker_site_json):
+def test_validate_pw(auth, expects, mock_bucket, mocker_site_json):
     mock_headers = {"Authorization": auth}
     event = {
         "headers": mock_headers,

tests/site_upload/test_fetch_upload_url.py

@@ -7,6 +7,7 @@
 from unittest import TestCase, mock
 
 from src.handlers.site_upload.fetch_upload_url import upload_url_handler
+from tests.utils import TEST_BUCKET, get_mock_metadata
 
 builtin_open = open
 
@@ -18,31 +19,34 @@ def mock_open(*args, **kwargs):
 
 
 def mock_json_load(*args):
-    return {"general": {"path": "s3://bucket/testpath"}}
-
-
-@mock_s3
-@mock.patch.dict(os.environ, {"BUCKET_NAME": "cumulus-aggregator-site-counts"})
-class TestFetchUploadUrl(TestCase):
-    def setUp(self):
-        self.bucket_name = "cumulus-aggregator-site-counts"
-        self.s3_client = boto3.client("s3", region_name="us-east-1")
-        self.s3_client.create_bucket(Bucket=self.bucket_name)
-
-    @mock.patch("builtins.open", mock_open)
-    @mock.patch("json.load", mock_json_load)
-    def test_fetch_upload_url(self):
-        body = json.dumps(
-            {"study": "covid", "subscription": "encounter", "filename": "covid.csv"}
-        )
-        context = {
-            "authorizer": {
-                "principalId": "general",
-            }
+    return {"general": {"path": f"s3://{TEST_BUCKET}/testpath"}}
+
+
+@mock.patch.dict(os.environ, {"BUCKET_NAME": TEST_BUCKET})
+@mock.patch("builtins.open", mock_open)
+@mock.patch("json.load", mock_json_load)
+@pytest.mark.parametrize(
+    "body,status",
+    [
+        (
+            {
+                "study": "covid",
+                "subscription": "encounter",
+                "filename": "encounter.parquet",
+            },
+            200,
+        ),
+        ({}, 500),
+    ],
+)
+def test_fetch_upload_url(body, status):
+    context = {
+        "authorizer": {
+            "principalId": "general",
         }
-        response = upload_url_handler({"body": body, "requestContext": context}, None)
-        self.assertEqual(response["statusCode"], 200)
-
-        body = json.dumps({})
-        response = upload_url_handler({"body": body}, None)
-        self.assertEqual(response["statusCode"], 500)
+    }
+    response = upload_url_handler(
+        {"body": json.dumps(body), "requestContext": context}, None
+    )
+    print(response)
+    assert response["statusCode"] == status