PRMP-571 - Lloyd George Ingestion - PDS Matching Updates Issues (#390)

* [PRMP-571] Add unicode utils method to check string starts_with and end_withs

* [PRMP-571] Add helper to assert test not raising expection

* [PRMP-571] Add test cases to reflect changes in PDS name match requirement

* [PRMP-571] Adding unit tests, replace try except catch block with `expect_not_to_raise`

* [PRMP-571] Amend implementation according to new requirements

* [PRMP-571] Add mock patients to test bulk upload with new requirements

* fix mistake in mock data (patient number not updated)

* [PRMP-571] Update name comparison logic at frontend

* edit mock patient to allow test with frontend

* fix mock patient 103 missing gp ods code

* set conftest.py to be exclude from coverage

* [PRMP-571] Add logic to select the most recent name for patient

* improve test coverage

app/src/helpers/test/testDataForPdsNameValidation.ts

@@ -0,0 +1,92 @@
+import { PatientDetails } from '../../types/generic/patientDetails';
+import { buildPatientDetails } from './testBuilders';
+import {
+    DOCUMENT_TYPE,
+    DOCUMENT_UPLOAD_STATE as documentUploadStates,
+    UploadDocument,
+} from '../../types/pages/UploadDocumentsPage/types';
+import { v4 as uuidv4 } from 'uuid';
+
+type PdsNameMatchingTestCase = {
+    patientDetails: PatientDetails;
+    patientNameInFileName: string;
+    shouldAcceptName: boolean;
+};
+
+type TestCaseJsonFormat = {
+    pds_name: {
+        family: string;
+        given: string[];
+    };
+    accept: string[];
+    reject: string[];
+};
+
+export const TEST_CASES_FOR_TWO_WORDS_FAMILY_NAME = {
+    pds_name: { family: 'Smith Anderson', given: ['Jane', 'Bob'] },
+    accept: ['Jane Bob Smith Anderson', 'Jane Smith Anderson', 'Jane B Smith Anderson'],
+    reject: ['Bob Smith Anderson', 'Jane Smith', 'Jane Anderson', 'Jane Anderson Smith'],
+};
+
+export const TEST_CASES_FOR_FAMILY_NAME_WITH_HYPHEN = {
+    pds_name: { family: 'Smith-Anderson', given: ['Jane'] },
+    accept: ['Jane Smith-Anderson'],
+    reject: ['Jane Smith Anderson', 'Jane Smith', 'Jane Anderson'],
+};
+
+export const TEST_CASES_FOR_TWO_WORDS_GIVEN_NAME = {
+    pds_name: { family: 'Smith', given: ['Jane Bob'] },
+    accept: ['Jane Bob Smith'],
+    reject: ['Jane Smith', 'Jane B Smith', 'Jane-Bob Smith', 'Bob Smith'],
+};
+
+export const TEST_CASES_FOR_TWO_WORDS_FAMILY_NAME_AND_GIVEN_NAME = {
+    pds_name: { family: 'Smith Anderson', given: ['Jane Bob'] },
+    accept: ['Jane Bob Smith Anderson'],
+    reject: [
+        'Jane Smith Anderson',
+        'Bob Smith Anderson',
+        'Jane B Smith Anderson',
+        'Jane Bob Smith',
+        'Jane Bob Anderson',
+    ],
+};
+
+export function loadTestCases(testCaseJson: TestCaseJsonFormat): Array<PdsNameMatchingTestCase> {
+    const patientDetails = buildPatientDetails({
+        givenName: testCaseJson['pds_name']['given'],
+        familyName: testCaseJson['pds_name']['family'],
+    });
+
+    const testCasesForAccept = testCaseJson['accept'].map((patientNameInFileName) => ({
+        patientDetails,
+        patientNameInFileName,
+        shouldAcceptName: true,
+    }));
+
+    const testCasesForReject = testCaseJson['reject'].map((patientNameInFileName) => ({
+        patientDetails,
+        patientNameInFileName,
+        shouldAcceptName: false,
+    }));
+
+    return [...testCasesForAccept, ...testCasesForReject];
+}
+
+export function buildLGUploadDocsFromFilenames(filenames: string[]): UploadDocument[] {
+    const fileObjects = filenames.map(
+        (filename) =>
+            new File(['test'], filename, {
+                type: 'application/pdf',
+            }),
+    );
+
+    return fileObjects.map((file) => ({
+        file,
+        state: documentUploadStates.SELECTED,
+        progress: 0,
+        id: uuidv4(),
+        docType: DOCUMENT_TYPE.LLOYD_GEORGE,
+        attempts: 0,
+    }));
+}

app/src/helpers/utils/uploadDocumentValidation.test.ts

@@ -0,0 +1,135 @@
+import { patientNameMatchesPds, uploadDocumentValidation } from './uploadDocumentValidation';
+import { buildPatientDetails } from '../test/testBuilders';
+import {
+    buildLGUploadDocsFromFilenames,
+    loadTestCases,
+    TEST_CASES_FOR_FAMILY_NAME_WITH_HYPHEN,
+    TEST_CASES_FOR_TWO_WORDS_FAMILY_NAME,
+    TEST_CASES_FOR_TWO_WORDS_FAMILY_NAME_AND_GIVEN_NAME,
+    TEST_CASES_FOR_TWO_WORDS_GIVEN_NAME,
+} from '../test/testDataForPdsNameValidation';
+import { UploadFilesErrors } from '../../types/pages/UploadDocumentsPage/types';
+
+describe('uploadDocumentValidation', () => {
+    describe('name validation', () => {
+        it('can handle a patient name with multiple words and special chars', () => {
+            const testPatient = buildPatientDetails({
+                givenName: ['Jane François', 'Bob'], // NFC
+                familyName: "O'Brian-Jones Anderson",
+                nhsNumber: '9000000009',
+                birthDate: '2011-09-19',
+            });
+            const patientNameInFile = "Jane François Bob O'Brian-Jones Anderson"; // NFD;
+            const testFileName = `1of1_Lloyd_George_Record_[${patientNameInFile}]_[9000000009]_[19-09-2011].pdf`;
+            const testUploadDocuments = buildLGUploadDocsFromFilenames([testFileName]);
+
+            const expected: UploadFilesErrors[] = [];
+            const actual = uploadDocumentValidation(testUploadDocuments, testPatient);
+
+            expect(actual).toEqual(expected);
+        });
+    });
+});
+
+describe('patientNameMatchesPds', () => {
+    it('returns true when the name in pds match patientNameInFileName', () => {
+        const patientDetails = buildPatientDetails({ givenName: ['Jane'], familyName: 'Smith' });
+        const patientNameInFileName = 'Jane Smith';
+        const expected: boolean = true;
+
+        const actual: boolean = patientNameMatchesPds(patientNameInFileName, patientDetails);
+        expect(actual).toBe(expected);
+    });
+
+    it('returns false when first name not match', () => {
+        const patientDetails = buildPatientDetails({ givenName: ['Jane'], familyName: 'Smith' });
+        const patientNameInFileName = 'Bob Smith';
+        const expected: boolean = false;
+
+        const actual: boolean = patientNameMatchesPds(patientNameInFileName, patientDetails);
+        expect(actual).toBe(expected);
+    });
+
+    it('returns false when last name not match', () => {
+        const patientDetails = buildPatientDetails({ givenName: ['Jane'], familyName: 'Smith' });
+        const patientNameInFileName = 'Jane Anderson';
+        const expected: boolean = false;
+
+        const actual: boolean = patientNameMatchesPds(patientNameInFileName, patientDetails);
+        expect(actual).toBe(expected);
+    });
+
+    it('should be case insensitive when comparing names', () => {
+        const patientDetails = buildPatientDetails({ givenName: ['jane'], familyName: 'SMITH' });
+        const patientNameInFileName = 'Jane Smith';
+        const expected: boolean = true;
+
+        const actual: boolean = patientNameMatchesPds(patientNameInFileName, patientDetails);
+        expect(actual).toBe(expected);
+    });
+
+    it('should be able to compare names with accent chars', () => {
+        const patientDetails = buildPatientDetails(
+            { givenName: ['Jàne'], familyName: 'Smïth' }, // NFD
+        );
+        const patientNameInFileName = 'Jàne Smïth'; // NFC
+        const expected: boolean = true;
+
+        const actual: boolean = patientNameMatchesPds(patientNameInFileName, patientDetails);
+        expect(actual).toBe(expected);
+    });
+
+    describe('Names with multiple words joined together', () => {
+        it.each(loadTestCases(TEST_CASES_FOR_TWO_WORDS_FAMILY_NAME))(
+            'from pds: ["Jane", "Bob"] Smith Anderson, filename: $patientNameInFileName, $shouldAcceptName',
+            ({ patientDetails, patientNameInFileName, shouldAcceptName }) => {
+                const actual: boolean = patientNameMatchesPds(
+                    patientNameInFileName,
+                    patientDetails,
+                );
+                const expected: boolean = shouldAcceptName;
+
+                expect(actual).toBe(expected);
+            },
+        );
+
+        it.each(loadTestCases(TEST_CASES_FOR_FAMILY_NAME_WITH_HYPHEN))(
+            'from pds: ["Jane"] Smith-Anderson, filename: $patientNameInFileName, $shouldAcceptName',
+            ({ patientDetails, patientNameInFileName, shouldAcceptName }) => {
+                const actual: boolean = patientNameMatchesPds(
+                    patientNameInFileName,
+                    patientDetails,
+                );
+                const expected: boolean = shouldAcceptName;
+
+                expect(actual).toBe(expected);
+            },
+        );
+
+        it.each(loadTestCases(TEST_CASES_FOR_TWO_WORDS_GIVEN_NAME))(
+            'from pds: ["Jane Bob"] Smith, filename: $patientNameInFileName, $shouldAcceptName',
+            ({ patientDetails, patientNameInFileName, shouldAcceptName }) => {
+                const actual: boolean = patientNameMatchesPds(
+                    patientNameInFileName,
+                    patientDetails,
+                );
+                const expected: boolean = shouldAcceptName;
+
+                expect(actual).toBe(expected);
+            },
+        );
+
+        it.each(loadTestCases(TEST_CASES_FOR_TWO_WORDS_FAMILY_NAME_AND_GIVEN_NAME))(
+            'from pds: ["Jane Bob"] Smith Anderson, filename: $patientNameInFileName, $shouldAcceptName',
+            ({ patientDetails, patientNameInFileName, shouldAcceptName }) => {
+                const actual: boolean = patientNameMatchesPds(
+                    patientNameInFileName,
+                    patientDetails,
+                );
+                const expected: boolean = shouldAcceptName;
+
+                expect(actual).toBe(expected);
+            },
+        );
+    });
+});

app/src/helpers/utils/uploadDocumentValidation.ts

@@ -8,7 +8,7 @@ for (let i = 0x300; i < 0x371; i++) {
     REGEX_ACCENT_MARKS_IN_NFD += String.fromCharCode(i);
 }
 const REGEX_ACCENT_CHARS_IN_NFC = 'À-ž';
-const REGEX_PATIENT_NAME_PATTERN = `[A-Za-z ${REGEX_ACCENT_CHARS_IN_NFC}${REGEX_ACCENT_MARKS_IN_NFD}]+`;
+const REGEX_PATIENT_NAME_PATTERN = `[A-Za-z ${REGEX_ACCENT_CHARS_IN_NFC}${REGEX_ACCENT_MARKS_IN_NFD}'-]+`;
 const REGEX_NHS_NUMBER_REGEX = '[0-9]{10}';
 const REGEX_LLOYD_GEORGE_FILENAME = new RegExp(
     `^[0-9]+of[0-9]+_Lloyd_George_Record_\\[(?<patient_name>${REGEX_PATIENT_NAME_PATTERN})]_\\[(?<nhs_number>${REGEX_NHS_NUMBER_REGEX})]_\\[(?<dob>\\d\\d-\\d\\d-\\d\\d\\d\\d)].pdf$`,
@@ -104,7 +104,6 @@ const validateWithPatientDetails = (
 ): UploadFilesErrors[] => {
     const dateOfBirth = new Date(patientDetails.birthDate);
     const dateOfBirthString = moment(dateOfBirth).format('DD-MM-YYYY');
-    const patientNameFromPds = [...patientDetails.givenName, patientDetails.familyName].join(' ');
     const nhsNumber = patientDetails.nhsNumber;
 
     const errors: UploadFilesErrors[] = [];
@@ -120,16 +119,24 @@ const validateWithPatientDetails = (
     }
 
     const patientNameInFilename = match?.groups?.patient_name as string;
-    if (!patientNameMatches(patientNameInFilename, patientNameFromPds)) {
+    if (!patientNameMatchesPds(patientNameInFilename, patientDetails)) {
         errors.push({ filename, error: fileUploadErrorMessages.patientNameError });
     }
 
     return errors;
 };
 
-const patientNameMatches = (patientNameInFileName: string, patientNameFromPds: string): boolean => {
-    return (
-        patientNameInFileName.normalize('NFD').toLowerCase() ===
-        patientNameFromPds.normalize('NFD').toLowerCase()
-    );
+export const patientNameMatchesPds = (
+    patientNameInFileName: string,
+    patientDetailsFromPds: PatientDetails,
+): boolean => {
+    const patientNameInFileNameNormalised = patientNameInFileName.normalize('NFD').toLowerCase();
+
+    const firstName = patientDetailsFromPds.givenName[0].normalize('NFD').toLowerCase();
+    const firstNameMatches = patientNameInFileNameNormalised.startsWith(firstName);
+
+    const familyName = patientDetailsFromPds.familyName.normalize('NFD').toLowerCase();
+    const familyNameMarches = patientNameInFileNameNormalised.endsWith(familyName);
+
+    return firstNameMatches && familyNameMarches;
 };

lambdas/models/pds_models.py

@@ -28,6 +28,19 @@ class Name(BaseModel):
     given: list[str] = [""]
     family: str
 
+    def is_currently_in_use(self) -> bool:
+        if not self.period:
+            return False
+        if self.use.lower() in ["nickname", "old"]:
+            return False
+
+        today = date.today()
+
+        name_started_already = self.period.start <= today
+        name_not_expired_yet = (not self.period.end) or self.period.end >= today
+
+        return name_started_already and name_not_expired_yet
+
 
 class Security(BaseModel):
     code: str
@@ -86,18 +99,31 @@ def is_unrestricted(self) -> bool:
         security = self.get_security()
         return security.code == "U"
 
-    def get_current_usual_name(self) -> Optional[Name]:
+    def get_usual_name(self) -> Optional[Name]:
         for entry in self.name:
             if entry.use.lower() == "usual":
                 return entry
 
+    def get_most_recent_name(self) -> Optional[Name]:
+        active_names = [name for name in self.name if name.is_currently_in_use()]
+        if not active_names:
+            return None
+
+        sorted_by_start_date_desc = sorted(
+            active_names, key=lambda name: name.period.start, reverse=True
+        )
+        return sorted_by_start_date_desc[0]
+
     def get_current_family_name_and_given_name(self) -> Tuple[str, list[str]]:
-        usual_name = self.get_current_usual_name()
-        if not usual_name:
-            logger.warning("The patient does not have a usual name.")
+        current_name = self.get_most_recent_name() or self.get_usual_name()
+        if not current_name:
+            logger.warning(
+                "The patient does not have a currently active name or a usual name."
+            )
             return "", [""]
-        given_name = usual_name.given
-        family_name = usual_name.family
+
+        given_name = current_name.given
+        family_name = current_name.family
 
         if not given_name or given_name == [""]:
             logger.warning("The given name of patient is empty.")