Feature/tools 10 process imported

src/tools/import_irap_tool.py

@@ -0,0 +1,113 @@
+from tools.models import IrapToolData, IrapToolDataImport, Tool
+
+
+def update_irap_data(old: IrapToolData, new: IrapToolDataImport) -> tuple[bool, {}, {}]:
+    # Copies all the existing values to the previous values dictionary,
+    # so they can be displayed to the tool admin when the changes
+    # are reviewed
+    # The format may not be the best, but it can be reviewed when
+    # it  has to be displayed
+
+    new_fields = new._meta.get_fields()
+    changed = False
+    previous_values = {}
+    for field in new_fields:
+        new_value = getattr(new, field.name)
+        old_value = getattr(old, field.name)
+        if old_value != new_value:
+            changed = True
+            setattr(old, field.name, new_value)
+        previous_values[field.name] = old_value
+
+    return changed, previous_values
+
+
+def are_field_identical(import_obj: IrapToolDataImport, old_values: dict) -> bool:
+    for key in old_values.keys():
+        if getattr(import_obj, key) != old_values[key]:
+            return False
+    return True
+
+
+def process_import():
+    """To be called after the irap data has been imported
+    successfully from Data Workspace.
+    There is no validation on the new records, as any validation will
+    happen when the data is imported from DW
+    """
+    IrapToolData.objects.update(imported=False)
+
+    imported_iraps = IrapToolDataImport.objects.all()
+    for imported_irap in imported_iraps:
+        irap, created = IrapToolData.objects.get_or_create(
+            product_irap_reference_number=imported_irap.product_irap_reference_number
+        )
+
+        if created:
+            # Easy case, a new record found
+            irap.product_name = imported_irap.product_name
+            irap.functionality = imported_irap.functionality
+            irap.after_import_status = IrapToolData.AfterImportStatus.NEW
+        else:
+            changed, changes = update_irap_data(irap, imported_irap)
+            match irap.after_import_status:
+                case IrapToolData.AfterImportStatus.REVIEWED:
+                    if changed:
+                        irap.after_import_status = (
+                            IrapToolData.AfterImportStatus.CHANGED
+                        )
+                        irap.previous_fields = changes
+
+                case IrapToolData.AfterImportStatus.DELETED:
+                    # This record was deleted at last import,
+                    # but the deletion was not reviewed
+                    irap.after_import_status = IrapToolData.AfterImportStatus.UNDELETED
+                    irap.previous_fields = changes
+
+                case IrapToolData.AfterImportStatus.CHANGED:
+                    # This record was changed at last import,
+                    # but the changes were not reviewed
+                    # check that the record has not been restored to what it was
+                    # if so, mark it as unchanged
+                    older_values = irap.previous_fields
+                    if are_field_identical(imported_irap, older_values):
+                        irap.after_import_status = (
+                            IrapToolData.AfterImportStatus.REVIEWED
+                        )
+                        irap.previous_fields = None
+        irap.imported = True
+        irap.save()
+
+    # Mark the deleted records: they exist in the irap table
+    # but don't exist in the imported data
+    deleted_iraps = IrapToolData.objects.filter(imported=False)
+    for deleted_irap in deleted_iraps:
+        if Tool.objects.filter(irap_tool=deleted_irap.pk):
+            # If this deleted record was used in the tool page,
+            # mark it as deleted and let the tool administrator
+            # handle the page.
+            # deleted_irap.update(
+            #     after_import_status=IrapToolData.AfterImportStatus.DELETED,
+            #     imported=True,
+            # )
+            deleted_irap.after_import_status = IrapToolData.AfterImportStatus.DELETED
+            deleted_irap.imported = True
+            deleted_irap.save()
+        else:
+            # If the deleted record was not used in the tool page,
+            # delete it. None will miss it!
+            deleted_irap.delete()
+
+
+def complete_irap_review(irap):
+    # This will be used after the tool administrator has checked
+    # the irap record.
+    # If the record was not available in the last import,
+    # it gets deleted
+    # Otherwise it is marked as 'REVIEWED'
+    if irap.after_import_status == IrapToolData.AfterImportStatus.DELETED:
+        irap.delete()
+    else:
+        irap.after_import_status = IrapToolData.AfterImportStatus.REVIEWED
+        irap.previous_fields = None
+        irap.savesave()

src/tools/test/test_irap_import.py

@@ -0,0 +1,209 @@
+from django.test import TestCase
+
+
+from tools.models import IrapToolDataImport, IrapToolData, Tool
+from tools.import_irap_tool import process_import
+
+RECORD_CREATED = 6
+FIRST_REFERENCE = 200
+
+
+class ImportIrapTestCase(TestCase):
+    def populate_table(self, model, record_to_create=RECORD_CREATED):
+        for i in range(0, record_to_create):
+            model.objects.create(
+                product_irap_reference_number=FIRST_REFERENCE + i,
+                product_name=f"Product name {i}",
+                functionality=f"Functionality {i}",
+            )
+
+    def test_new(self):
+        self.populate_table(IrapToolDataImport)
+        self.assertEqual(IrapToolDataImport.objects.all().count(), RECORD_CREATED)
+        self.assertEqual(IrapToolData.objects.all().count(), 0)
+        process_import()
+        self.assertEqual(IrapToolDataImport.objects.all().count(), RECORD_CREATED)
+        records_in_irap_table = IrapToolData.objects.all().count()
+        self.assertEqual(
+            records_in_irap_table,
+            RECORD_CREATED,
+        )
+        self.assertEqual(
+            IrapToolData.objects.filter(
+                after_import_status=IrapToolData.AfterImportStatus.NEW
+            ).count(),
+            RECORD_CREATED,
+        )
+
+    def test_update(self):
+        self.populate_table(IrapToolDataImport)
+        self.populate_table(IrapToolData)
+        IrapToolData.objects.update(
+            after_import_status=IrapToolData.AfterImportStatus.REVIEWED
+        )
+        old_product_name = "New name"
+        # Change a record in one table, to see if the changes get picked
+        update_record = IrapToolData.objects.get(
+            product_irap_reference_number=FIRST_REFERENCE
+        )
+        update_record.product_name = old_product_name
+        update_record.save()
+        process_import()
+        # There should be one record  flagged changed
+        self.assertEqual(
+            IrapToolData.objects.filter(
+                after_import_status=IrapToolData.AfterImportStatus.CHANGED
+            ).count(),
+            1,
+        )
+        self.assertEqual(
+            IrapToolData.objects.filter(
+                after_import_status=IrapToolData.AfterImportStatus.REVIEWED
+            ).count(),
+            RECORD_CREATED - 1,
+        )
+
+        changed_obj = IrapToolData.objects.get(
+            after_import_status=IrapToolData.AfterImportStatus.CHANGED
+        )
+        self.assertEqual(changed_obj.product_irap_reference_number, FIRST_REFERENCE)
+        self.assertNotEqual(changed_obj.product_name, old_product_name)
+
+    def test_delete(self):
+        self.populate_table(IrapToolDataImport, RECORD_CREATED - 1)
+        self.populate_table(IrapToolData)
+        IrapToolData.objects.update(
+            after_import_status=IrapToolData.AfterImportStatus.REVIEWED
+        )
+
+        self.assertEqual(
+            IrapToolData.objects.all().count(),
+            RECORD_CREATED,
+        )
+        process_import()
+
+        # There should be one less record
+        self.assertEqual(
+            IrapToolData.objects.all().count(),
+            RECORD_CREATED - 1,
+        )
+
+    def test_soft_delete(self):
+        self.populate_table(IrapToolDataImport)
+        self.populate_table(IrapToolData)
+        IrapToolData.objects.update(
+            after_import_status=IrapToolData.AfterImportStatus.REVIEWED
+        )
+        # Remove one record from the import table
+        product_irap_reference_number_to_be_deleted = FIRST_REFERENCE
+        IrapToolDataImport.objects.get(
+            product_irap_reference_number=product_irap_reference_number_to_be_deleted
+        ).delete()
+        self.assertEqual(
+            IrapToolDataImport.objects.all().count(),
+            RECORD_CREATED - 1,
+        )
+        # Create a pge referencing the tool to be deleted
+        obj_to_be_deleted = IrapToolData.objects.get(
+            product_irap_reference_number=product_irap_reference_number_to_be_deleted
+        )
+        Tool.objects.create(
+            irap_tool=obj_to_be_deleted,
+            depth=1,
+            title="Title",
+            path="Path",
+        )
+
+        self.assertEqual(
+            IrapToolData.objects.all().count(),
+            RECORD_CREATED,
+        )
+        process_import()
+
+        # No record has been deleted
+        self.assertEqual(
+            IrapToolData.objects.all().count(),
+            RECORD_CREATED,
+        )
+        #         But there is a record marked deleted
+        self.assertEqual(
+            IrapToolData.objects.filter(
+                after_import_status=IrapToolData.AfterImportStatus.DELETED
+            ).count(),
+            1,
+        )
+
+    def test_undelete(self):
+        self.populate_table(IrapToolDataImport)
+        self.populate_table(IrapToolData)
+        IrapToolData.objects.update(
+            after_import_status=IrapToolData.AfterImportStatus.REVIEWED
+        )
+        IrapToolData.objects.filter(
+            product_irap_reference_number=FIRST_REFERENCE
+        ).update(after_import_status=IrapToolData.AfterImportStatus.DELETED)
+
+        process_import()
+
+        # No record has been deleted
+        self.assertEqual(
+            IrapToolData.objects.all().count(),
+            RECORD_CREATED,
+        )
+        undeleted_objs = IrapToolData.objects.filter(
+            after_import_status=IrapToolData.AfterImportStatus.UNDELETED
+        )
+        #         But there is a record marked undeleted
+        self.assertEqual(
+            undeleted_objs.count(),
+            1,
+        )
+
+        self.assertEqual(
+            undeleted_objs.first().product_irap_reference_number,
+            FIRST_REFERENCE,
+        )
+
+    def test_record_reset(self):
+        # Only create one record in both tables
+        IrapToolDataImport.objects.create(
+            product_irap_reference_number=FIRST_REFERENCE,
+            product_name="Product name changes",
+            functionality="Functionality",
+        )
+        IrapToolData.objects.create(
+            product_irap_reference_number=FIRST_REFERENCE,
+            product_name="Product name original",
+            functionality="Functionality",
+            after_import_status=IrapToolData.AfterImportStatus.REVIEWED,
+        )
+        process_import()
+        irap_tool_data = IrapToolData.objects.get(
+            product_irap_reference_number=FIRST_REFERENCE
+        )
+
+        self.assertEqual(
+            irap_tool_data.after_import_status, IrapToolData.AfterImportStatus.CHANGED
+        )
+        self.assertEqual(irap_tool_data.product_name, "Product name changes")
+        self.assertNotEqual(irap_tool_data.previous_fields, None)
+        IrapToolDataImport.objects.filter(
+            product_irap_reference_number=FIRST_REFERENCE,
+        ).update(
+            product_name=f"Product name original",
+        )
+
+        process_import()
+        irap_tool_data = IrapToolData.objects.get(
+            product_irap_reference_number=FIRST_REFERENCE
+        )
+
+        self.assertEqual(irap_tool_data.product_name, "Product name original")
+        # self.assertEqual(
+        #     irap_tool_data.previous_fields,
+        #     None
+        # )
+
+        self.assertEqual(
+            irap_tool_data.after_import_status, IrapToolData.AfterImportStatus.REVIEWED
+        )