PRMP-1367

lambdas/enums/feature_flags.py

@@ -7,3 +7,4 @@ class FeatureFlags(Enum):
     UPLOAD_LAMBDA_ENABLED = "uploadLambdaEnabled"
     UPLOAD_ARF_WORKFLOW_ENABLED = "uploadArfWorkflowEnabled"
     USE_SMARTCARD_AUTH = "useSmartcardAuth"
+    LLOYD_GEORGE_VALIDATION_STRICT_MODE = "lloydGeorgeValidationStrictMode"

lambdas/enums/validation_score.py

@@ -0,0 +1,17 @@
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class ValidationScore(Enum):
+    NO_MATCH = 0
+    PARTIAL_MATCH = 1
+    MIXED_FULL_MATCH = 2
+    FULL_MATCH = 3
+
+
+class ValidationResult(BaseModel):
+    given_name_match: list[str] = []
+    family_name_match: Optional[str] = None
+    score: ValidationScore = ValidationScore.NO_MATCH

lambdas/handlers/bulk_upload_handler.py

@@ -1,4 +1,6 @@
+from enums.feature_flags import FeatureFlags
 from services.bulk_upload_service import BulkUploadService
+from services.feature_flags_service import FeatureFlagService
 from utils.audit_logging_setup import LoggingService
 from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions
 from utils.decorators.override_error_check import override_error_check
@@ -14,6 +16,16 @@
 @handle_lambda_exceptions
 def lambda_handler(event, _context):
     logger.info("Received event. Starting bulk upload process")
+    feature_flag_service = FeatureFlagService()
+    validation_strict_mode_flag_object = feature_flag_service.get_feature_flags_by_flag(
+        FeatureFlags.LLOYD_GEORGE_VALIDATION_STRICT_MODE.value
+    )
+    validation_strict_mode = validation_strict_mode_flag_object[
+        FeatureFlags.LLOYD_GEORGE_VALIDATION_STRICT_MODE.value
+    ]
+
+    if validation_strict_mode:
+        logger.info("Lloyd George validation strict mose is enabled")
 
     if "Records" not in event or len(event["Records"]) < 1:
         http_status_code = 400
@@ -25,7 +37,7 @@ def lambda_handler(event, _context):
             status_code=http_status_code, body=response_body, methods="GET"
         ).create_api_gateway_response()
 
-    bulk_upload_service = BulkUploadService()
+    bulk_upload_service = BulkUploadService(strict_mode=validation_strict_mode)
 
     try:
         bulk_upload_service.process_message_queue(event["Records"])

lambdas/models/pds_models.py

@@ -116,26 +116,28 @@ def get_usual_name(self) -> Optional[Name]:
             if entry.use.lower() == "usual":
                 return entry
 
-    def get_most_recent_name(self) -> Optional[Name]:
-        active_names = [name for name in self.name if name.is_currently_in_use()]
-        if not active_names:
-            return None
-
+    def get_names_by_start_date(self) -> [Name]:
         sorted_by_start_date_desc = sorted(
-            active_names, key=lambda name: name.period.start, reverse=True
+            self.name,
+            key=lambda name: (
+                (1, name.period.start, name.use.lower() == "usual")
+                if name.period and name.period.start is not None
+                else (0, 0, name.use.lower() == "usual")
+            ),
+            reverse=True,
         )
-        return sorted_by_start_date_desc[0]
+        return sorted_by_start_date_desc
 
     def get_current_family_name_and_given_name(self) -> Tuple[str, list[str]]:
-        current_name = self.get_most_recent_name() or self.get_usual_name()
-        if not current_name:
+        ordered_names = self.get_names_by_start_date()
+        if not ordered_names:
             logger.warning(
                 "The patient does not have a currently active name or a usual name."
             )
             return "", [""]
 
-        given_name = current_name.given
-        family_name = current_name.family
+        given_name = ordered_names[0].given
+        family_name = ordered_names[0].family
 
         if not given_name or given_name == [""]:
             logger.warning("The given name of patient is empty.")

lambdas/services/bulk_upload_service.py

@@ -34,7 +34,8 @@
     LGInvalidFilesException,
     allowed_to_ingest_ods_code,
     getting_patient_info_from_pds,
-    validate_filename_with_patient_details,
+    validate_filename_with_patient_details_lenient,
+    validate_filename_with_patient_details_strict,
     validate_lg_file_names,
 )
 from utils.request_context import request_context
@@ -48,11 +49,11 @@
 
 
 class BulkUploadService:
-    def __init__(self):
+    def __init__(self, strict_mode):
         self.dynamo_repository = BulkUploadDynamoRepository()
         self.sqs_repository = BulkUploadSqsRepository()
         self.s3_repository = BulkUploadS3Repository()
-
+        self.strict_mode = strict_mode
         self.pdf_content_type = "application/pdf"
         self.unhandled_messages = []
         self.file_path_cache = {}
@@ -133,13 +134,27 @@ def handle_sqs_message(self, message: dict):
             patient_ods_code = (
                 pds_patient_details.get_ods_code_or_inactive_status_for_gp()
             )
-            is_name_validation_based_on_historic_name = (
-                validate_filename_with_patient_details(file_names, pds_patient_details)
-            )
+            if not self.strict_mode:
+                (
+                    name_validation_accepted_reason,
+                    is_name_validation_based_on_historic_name,
+                ) = validate_filename_with_patient_details_lenient(
+                    file_names, pds_patient_details
+                )
+                accepted_reason = self.concatenate_acceptance_reason(
+                    accepted_reason, name_validation_accepted_reason
+                )
+            else:
+                is_name_validation_based_on_historic_name = (
+                    validate_filename_with_patient_details_strict(
+                        file_names, pds_patient_details
+                    )
+                )
             if is_name_validation_based_on_historic_name:
                 accepted_reason = self.concatenate_acceptance_reason(
                     accepted_reason, "Patient matched on historical name"
                 )
+
             if not allowed_to_ingest_ods_code(patient_ods_code):
                 raise LGInvalidFilesException("Patient not registered at your practice")
             patient_death_notification_status = (

lambdas/services/edge_presign_service.py

@@ -25,7 +25,7 @@ def use_presign(self, request_values: dict):
         domain_name: str = request_values["domain_name"]
 
         presign_string: str = f"{uri}?{querystring}"
-        encoded_presign_string: str = presign_string.encode("utf-8")
+        encoded_presign_string = presign_string.encode("utf-8")
         presign_credentials_hash: str = hashlib.md5(encoded_presign_string).hexdigest()
 
         self.attempt_presign_ingestion(
@@ -70,7 +70,7 @@ def filter_request_values(request: dict) -> dict:
             uri: str = request["uri"]
             querystring: str = request["querystring"]
             headers: dict = request["headers"]
-            origin: str = request.get("origin", {})
+            origin: dict = request.get("origin", {})
             domain_name: str = origin["s3"]["domainName"]
         except KeyError as e:
             logger.error(f"Missing request component: {str(e)}")

lambdas/services/mock_data/pds_patient_9000000003_H85686_gp.json

@@ -38,7 +38,7 @@
             "use": "usual",
             "period": {
                 "start": "2020-01-01",
-                "end": "2025-12-31"
+                "end": "2035-12-31"
             },
             "given": [
                 "Jane"
@@ -55,8 +55,8 @@
             "id": "1234",
             "use": "other",
             "period": {
-                "start": "2020-01-01",
-                "end": "2025-12-31"
+                "start": "2010-01-01",
+                "end": "2019-12-31"
             },
             "given": [
                 "Jim"

lambdas/tests/unit/handlers/conftest.py

@@ -111,3 +111,12 @@ def mock_upload_lambda_disabled(mocker):
         "uploadLambdaEnabled": False
     }
     yield mock_upload_lambda_feature_flag
+
+
+@pytest.fixture
+def mock_validation_strict_disabled(mocker):
+    mock_function = mocker.patch.object(FeatureFlagService, "get_feature_flags_by_flag")
+    mock_upload_lambda_feature_flag = mock_function.return_value = {
+        "lloydGeorgeValidationStrictMode": False
+    }
+    yield mock_upload_lambda_feature_flag

lambdas/tests/unit/handlers/test_bulk_upload_handler.py

@@ -15,7 +15,9 @@ def mock_service(mocker):
     yield mocker.patch.object(BulkUploadService, "process_message_queue")
 
 
-def test_can_process_event_with_one_message(mock_service, context, set_env):
+def test_can_process_event_with_one_message(
+    mock_service, context, set_env, mock_validation_strict_disabled
+):
     expected = ApiGatewayResponse(
         200, "Finished processing all 1 messages", "GET"
     ).create_api_gateway_response()
@@ -24,7 +26,9 @@ def test_can_process_event_with_one_message(mock_service, context, set_env):
     assert expected == actual
 
 
-def test_can_process_event_with_several_messages(mock_service, context, set_env):
+def test_can_process_event_with_several_messages(
+    mock_service, context, set_env, mock_validation_strict_disabled
+):
     expected = ApiGatewayResponse(
         200, "Finished processing all 3 messages", "GET"
     ).create_api_gateway_response()
@@ -34,7 +38,7 @@ def test_can_process_event_with_several_messages(mock_service, context, set_env)
 
 
 def test_receive_correct_response_when_service_returns_error(
-    mock_service, context, set_env
+    mock_service, context, set_env, mock_validation_strict_disabled
 ):
     expected = ApiGatewayResponse(
         500, "Bulk upload failed with error: ", "GET"
@@ -46,7 +50,7 @@ def test_receive_correct_response_when_service_returns_error(
 
 
 def test_receive_correct_response_when_no_records_in_event(
-    mock_service, context, set_env
+    mock_service, context, set_env, mock_validation_strict_disabled
 ):
     expected = ApiGatewayResponse(
         400,

lambdas/tests/unit/helpers/data/pds/test_cases_for_patient_name_matching.py

@@ -18,15 +18,22 @@ class PdsNameMatchingTestCase(NamedTuple):
         "Jane Bob Smith Anderson",
         "Jane Smith Anderson",
         "Jane B Smith Anderson",
+        "Bob Smith Anderson",
     ],
     "reject": [
-        "Bob Smith Anderson",
         "Jane Smith",
         "Jane Anderson",
         "Jane Anderson Smith",
     ],
 }
 
+TEST_CASES_FOR_TWO_WORDS_FAMILY_NAME_STRICT = copy.deepcopy(
+    TEST_CASES_FOR_TWO_WORDS_FAMILY_NAME
+)
+TEST_CASES_FOR_TWO_WORDS_FAMILY_NAME_STRICT["reject"].append(
+    TEST_CASES_FOR_TWO_WORDS_FAMILY_NAME_STRICT["accept"].pop(3)
+)
+
 TEST_CASES_FOR_FAMILY_NAME_WITH_HYPHEN = {
     "pds_name": {"family": "Smith-Anderson", "given": ["Jane"]},
     "accept": ["Jane Smith-Anderson"],

lambdas/tests/unit/models/test_pds_models.py

@@ -253,6 +253,19 @@ def test_get_current_family_name_and_given_name_return_the_first_usual_name_if_a
     assert actual == expected_given_name
 
 
+def test_get_current_family_name_and_given_name_return_the_first_most_recent_name():
+    name_1 = build_test_name(use="temp", start="1990-01-01", end=None, given=["Jones"])
+    name_2 = build_test_name(use="usual", start=None, end=None, given=["Jane"])
+    name_3 = build_test_name(use="usual", start=None, end=None, given=["Bob"])
+
+    test_patient = build_test_patient_with_names([name_1, name_2, name_3])
+
+    expected_given_name = ["Jones"]
+    actual = test_patient.get_patient_details(test_patient.id).given_name
+
+    assert actual == expected_given_name
+
+
 def test_get_current_family_name_and_given_name_logs_a_warning_if_no_current_name_or_usual_name_found(
     caplog,
 ):
@@ -330,7 +343,7 @@ def test_name_is_currently_in_use_return_false_for_nickname_or_old_name():
 
 
 @freeze_time("2024-01-01")
-def test_get_most_recent_name_return_the_name_with_most_recent_start_date():
+def test_get_names_by_start_date_return_ordered_list_by_start_date():
     name_1 = build_test_name(start="1990-01-01", end=None, given=["Jane"])
     name_2 = build_test_name(start="2010-02-14", end=None, given=["Jones"])
     name_3 = build_test_name(start="2000-03-25", end=None, given=["Bob"])
@@ -341,22 +354,31 @@ def test_get_most_recent_name_return_the_name_with_most_recent_start_date():
         use="nickname", start="2023-01-01", end=None, given=["Janie"]
     )
     future_name = build_test_name(start="2047-01-01", end=None, given=["Neo Jane"])
+    name_no_date = build_test_name(start=None, end=None, given=["Jane"])
 
     test_patient = build_test_patient_with_names(
-        [name_1, name_2, name_3, expired_name, nickname, future_name]
+        [name_1, name_2, name_3, expired_name, nickname, future_name, name_no_date]
     )
 
-    expected = name_2
-    actual = test_patient.get_most_recent_name()
+    expected = [
+        future_name,
+        nickname,
+        expired_name,
+        name_2,
+        name_3,
+        name_1,
+        name_no_date,
+    ]
+    actual = test_patient.get_names_by_start_date()
 
     assert actual == expected
 
 
-@freeze_time("2024-01-01")
-def test_get_most_recent_name_return_none_if_no_active_name_found():
-    test_patient = build_test_patient_with_names([])
-
-    assert test_patient.get_most_recent_name() is None
+# @freeze_time("2024-01-01")
+# def test_get_most_recent_name_return_none_if_no_active_name_found():
+#     test_patient = build_test_patient_with_names([])
+#
+#     assert test_patient.get_names_by_start_date() is None
 
 
 def test_get_death_notification_status_return_the_death_notification_status():