Refactors test_ingest_events.py removing Mocks

packages/flare/tests/bin/conftest.py

@@ -0,0 +1,139 @@
+import json
+import os
+import pytest
+import sys
+
+from datetime import datetime
+from pytest import FixtureRequest
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../bin"))
+from constants import KV_COLLECTION_NAME
+
+
+class FakeStoragePassword:
+    def __init__(self, username: str, clear_password: str) -> None:
+        self._state = {
+            "username": username,
+            "clear_password": clear_password,
+        }
+
+    @property
+    def content(self: "FakeStoragePassword") -> "FakeStoragePassword":
+        return self
+
+    @property
+    def username(self) -> str:
+        return self._state["username"]
+
+    @property
+    def clear_password(self) -> str:
+        return self._state["clear_password"]
+
+
+class FakeStoragePasswords:
+    def __init__(self, passwords: List[FakeStoragePassword]) -> None:
+        self._passwords = passwords
+
+    def list(self) -> List[FakeStoragePassword]:
+        return self._passwords
+
+
+class FakeKVStoreCollectionData:
+    def __init__(self) -> None:
+        self._data: dict[str, str] = {}
+
+    def insert(self, data: str) -> dict[str, str]:
+        entry = json.loads(data)
+        self._data[entry["_key"]] = entry["value"]
+        return entry
+
+    def update(self, id: str, data: str) -> dict[str, str]:
+        entry = json.loads(data)
+        self._data[id] = entry["value"]
+        return entry
+
+    def query(self, **query: dict) -> List[Dict[str, str]]:
+        return [{"_key": key, "value": value} for key, value in self._data.items()]
+
+
+class FakeKVStoreCollection:
+    def __init__(self) -> None:
+        self._data = FakeKVStoreCollectionData()
+
+    @property
+    def data(self) -> FakeKVStoreCollectionData:
+        return self._data
+
+
+class FakeKVStoreCollections:
+    def __init__(self) -> None:
+        self._collections: dict[str, Any] = {}
+
+    def __getitem__(self, key: str) -> FakeKVStoreCollection:
+        return self._collections[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self._collections
+
+    def create(self, name: str, fields: dict) -> dict[str, Any]:
+        self._collections[name] = FakeKVStoreCollection()
+        return {"headers": {}, "reason": "Created", "status": 200, "body": ""}
+
+
+class FakeLogger:
+    def __init__(self) -> None:
+        self.messages: List[str] = []
+
+    def info(self, message: str) -> None:
+        self.messages.append(f"INFO: {message}")
+
+    def error(self, message: str) -> None:
+        self.messages.append(f"ERROR: {message}")
+
+
+class FakeFlareAPI:
+    def __init__(self, api_key: str, tenant_id: int) -> None:
+        self.api_key = api_key
+        self.tenant_id = tenant_id
+
+    def fetch_feed_events(
+        self,
+        next: Optional[str],
+        start_date: Optional[datetime],
+        ingest_metadata_only: bool,
+    ) -> List[tuple[dict, str]]:
+        return [({"event": "test_event"}, "next_token")]
+
+
+@pytest.fixture
+def storage_passwords(request: FixtureRequest) -> FakeStoragePasswords:
+    passwords: list[FakeStoragePassword] = []
+    data: list[tuple[str, str]] = request.param if hasattr(request, "param") else []
+
+    if data:
+        for item in data:
+            passwords.append(
+                FakeStoragePassword(username=item[0], clear_password=item[1])
+            )
+
+    return FakeStoragePasswords(passwords=passwords)
+
+
+@pytest.fixture
+def kvstore(request: FixtureRequest) -> FakeKVStoreCollections:
+    kvstore = FakeKVStoreCollections()
+    data: list[tuple[str, str]] = request.param if hasattr(request, "param") else []
+
+    if data:
+        kvstore.create(name=KV_COLLECTION_NAME, fields={})
+        for item in data:
+            kvstore[KV_COLLECTION_NAME].data.insert(
+                json.dumps({"_key": item[0], "value": item[1]})
+            )
+
+    return kvstore

packages/flare/tests/bin/test_ingest_events.py

@@ -1,15 +1,14 @@
-import json
 import os
 import pytest
 import sys
 
+from conftest import FakeKVStoreCollections
+from conftest import FakeStoragePasswords
 from datetime import date
 from datetime import datetime
 from datetime import timedelta
 from typing import Any
 from unittest.mock import MagicMock
-from unittest.mock import Mock
-from unittest.mock import PropertyMock
 from unittest.mock import call
 from unittest.mock import patch
 
@@ -18,6 +17,7 @@
 from constants import CRON_JOB_THRESHOLD_SINCE_LAST_FETCH
 from constants import KV_COLLECTION_NAME
 from constants import CollectionKeys
+from constants import PasswordKeys
 from cron_job_ingest_events import fetch_feed
 from cron_job_ingest_events import get_api_key
 from cron_job_ingest_events import get_collection_value
@@ -30,128 +30,120 @@
 from cron_job_ingest_events import save_last_ingested_tenant_id
 
 
-def test_get_collection_value_expect_none() -> None:
-    kvstore = MagicMock()
+def test_get_collection_value_expect_none(kvstore: FakeKVStoreCollections) -> None:
     assert get_collection_value(kvstore=kvstore, key="some_key") is None
 
 
-def test_get_collection_value_expect_result() -> None:
-    kvstore = MagicMock()
-    kvstore.__contains__.side_effect = lambda x: x == KV_COLLECTION_NAME
-    kvstore[KV_COLLECTION_NAME].data.query.return_value = [
-        {
-            "_key": "some_key",
-            "value": "some_value",
-        },
-    ]
-
+@pytest.mark.parametrize("kvstore", [[("some_key", "some_value")]], indirect=True)
+def test_get_collection_value_expect_result(kvstore: FakeKVStoreCollections) -> None:
     assert get_collection_value(kvstore=kvstore, key="some_key") == "some_value"
 
 
-def test_save_collection_value_expect_insert() -> None:
-    key = "some_key"
-    value = "some_value"
-    kvstore = MagicMock()
-    save_collection_value(kvstore=kvstore, key=key, value=value)
-    kvstore[KV_COLLECTION_NAME].data.insert.assert_called_once_with(
-        json.dumps({"_key": key, "value": value})
-    )
+def test_save_collection_value_expect_insert(kvstore: FakeKVStoreCollections) -> None:
+    save_collection_value(kvstore=kvstore, key="some_key", value="some_value")
+    assert kvstore[KV_COLLECTION_NAME].data.query() == [
+        {"_key": "some_key", "value": "some_value"}
+    ]
 
 
-def test_save_collection_value_expect_update() -> None:
+@pytest.mark.parametrize("kvstore", [[("some_key", "old_value")]], indirect=True)
+def test_save_collection_value_expect_update(kvstore: FakeKVStoreCollections) -> None:
     key = "some_key"
     value = "update_value"
-    kvstore = MagicMock()
-    kvstore.__contains__.side_effect = lambda x: x == KV_COLLECTION_NAME
-    kvstore[KV_COLLECTION_NAME].data.query.return_value = [
-        {
-            "_key": key,
-            "value": "old_value",
-        },
+
+    assert kvstore[KV_COLLECTION_NAME].data.query() == [
+        {"_key": key, "value": "old_value"}
     ]
     save_collection_value(kvstore=kvstore, key=key, value=value)
-    kvstore[KV_COLLECTION_NAME].data.update.assert_called_once_with(
-        id=key,
-        data=json.dumps({"value": value}),
-    )
+    assert kvstore[KV_COLLECTION_NAME].data.query() == [{"_key": key, "value": value}]
 
 
-def test_get_api_key_tenant_id_expect_exception() -> None:
-    storage_passwords = MagicMock()
-
+@pytest.mark.parametrize("storage_passwords", [[]], indirect=True)
+def test_get_api_key_expect_exception(storage_passwords: FakeStoragePasswords) -> None:
     with pytest.raises(Exception, match="API key not found"):
         get_api_key(storage_passwords=storage_passwords)
 
+
+@pytest.mark.parametrize(
+    "storage_passwords",
+    [[(PasswordKeys.API_KEY.value, "some_api_key")]],
+    indirect=True,
+)
+def test_tenant_id_expect_exception(storage_passwords: FakeStoragePasswords) -> None:
     with pytest.raises(Exception, match="Tenant ID not found"):
         get_tenant_id(storage_passwords=storage_passwords)
 
 
-def test_get_api_credentials_expect_api_key_and_tenant_id() -> None:
-    storage_passwords = MagicMock()
-
-    api_key_item = Mock()
-    type(api_key_item.content).username = PropertyMock(return_value="api_key")
-    type(api_key_item).clear_password = PropertyMock(return_value="some_api_key")
-
-    tenant_id_item = Mock()
-    type(tenant_id_item.content).username = PropertyMock(return_value="tenant_id")
-    type(tenant_id_item).clear_password = PropertyMock(return_value=11111)
-
-    storage_passwords.list.return_value = [api_key_item, tenant_id_item]
-
-    api_key = get_api_key(storage_passwords=storage_passwords)
-    assert api_key == "some_api_key"
-    tenant_id = get_tenant_id(storage_passwords=storage_passwords)
-    assert tenant_id == 11111
+@pytest.mark.parametrize(
+    "storage_passwords",
+    [
+        [
+            (PasswordKeys.API_KEY.value, "some_api_key"),
+            (PasswordKeys.TENANT_ID.value, 11111),
+        ],
+    ],
+    indirect=True,
+)
+def test_get_api_credentials_expect_api_key_and_tenant_id(
+    storage_passwords: FakeStoragePasswords,
+) -> None:
+    assert get_api_key(storage_passwords=storage_passwords) == "some_api_key"
+    assert get_tenant_id(storage_passwords=storage_passwords) == 11111
 
 
-@patch(
-    "cron_job_ingest_events.get_collection_value", return_value="not_an_isoformat_date"
+@pytest.mark.parametrize(
+    "kvstore",
+    [[(CollectionKeys.START_DATE.value, "non_date_parseable_value")]],
+    indirect=True,
 )
-def test_get_start_date_expect_none(get_collection_value_mock: MagicMock) -> None:
-    kvstore = MagicMock()
+def test_get_start_date_expect_none(kvstore: FakeKVStoreCollections) -> None:
     assert get_start_date(kvstore=kvstore) is None
 
 
-@patch(
-    "cron_job_ingest_events.get_collection_value", return_value=date.today().isoformat()
+@pytest.mark.parametrize(
+    "kvstore",
+    [[(CollectionKeys.START_DATE.value, "2000-01-01")]],
+    indirect=True,
 )
-def test_get_start_date_expect_date(get_collection_value_mock: MagicMock) -> None:
-    kvstore = MagicMock()
-    assert isinstance(get_start_date(kvstore), date)
+def test_get_start_date_expect_date(kvstore: FakeKVStoreCollections) -> None:
+    assert get_start_date(kvstore) == date(2000, 1, 1)
 
 
-@patch("cron_job_ingest_events.get_collection_value", return_value="not_a_number")
 def test_get_last_ingested_tenant_id_expect_none(
-    get_collection_value_mock: MagicMock,
+    kvstore: FakeKVStoreCollections,
 ) -> None:
-    kvstore = MagicMock()
     assert get_last_ingested_tenant_id(kvstore=kvstore) is None
 
 
-@patch("cron_job_ingest_events.get_collection_value", return_value="11111")
+@pytest.mark.parametrize(
+    "kvstore",
+    [[(CollectionKeys.LAST_INGESTED_TENANT_ID.value, 11111)]],
+    indirect=True,
+)
 def test_get_last_ingested_tenant_id_expect_integer(
-    get_collection_value_mock: MagicMock,
+    kvstore: FakeKVStoreCollections,
 ) -> None:
-    kvstore = MagicMock()
     assert get_last_ingested_tenant_id(kvstore=kvstore) == 11111
 
 
-@patch(
-    "cron_job_ingest_events.get_collection_value", return_value="not_an_isoformat_date"
-)
-def test_get_last_fetched_expect_none(get_collection_value_mock: MagicMock) -> None:
-    kvstore = MagicMock()
+def test_get_last_fetched_expect_none(kvstore: FakeKVStoreCollections) -> None:
     assert get_last_fetched(kvstore=kvstore) is None
 
 
-@patch(
-    "cron_job_ingest_events.get_collection_value",
-    return_value=datetime.now().isoformat(),
+@pytest.mark.parametrize(
+    "kvstore",
+    [
+        [
+            (
+                CollectionKeys.TIMESTAMP_LAST_FETCH.value,
+                datetime(2000, 1, 1, 12, 0, 0).isoformat(),
+            )
+        ]
+    ],
+    indirect=True,
 )
-def test_get_last_fetched_expect_datetime(get_collection_value_mock: MagicMock) -> None:
-    kvstore = MagicMock()
-    assert isinstance(get_last_fetched(kvstore=kvstore), datetime)
+def test_get_last_fetched_expect_datetime(kvstore: FakeKVStoreCollections) -> None:
+    assert get_last_fetched(kvstore=kvstore) == datetime(2000, 1, 1, 12, 0, 0)
 
 
 @patch("cron_job_ingest_events.save_collection_value")