New importer - rebase of #33 (#73)

* Add the gist of a new importer Extracted from internal project * Convert mock airtable to a callable This means the data in it is pure, and is safe to mutate if needed. * Update tests for importer * Use mocks for stubbing out airtable * Move mock airtable into tests dir Reduce the size of the package, and put test fixtures where they should be * Remove display method Woops! * Add tests for creating an object * Fix count reporting on index view * Reinstate ability to pass multiple models to import_airtable command * Reinstate fix for m2m on model creation (from #36) * Fix "null has no effect on ManyToManyField" warning on tests Also rename the migration that adds the advert-publications relation to something more meaningful * Add test for posting to airtable_import_listing * Test (and fix) importing of page models * Remove unused wagtail_route template tag This was introduced in a4298fc to make the overridden type_index.html template work with URL routes for both Wagtail 3.2 and 4.0, and became redundant in 431fd47 when the template was replaced with the one from Wagtail 4.2 (which doesn't refer to those routes at all, as they're passed in the template context). * Update docs to state that PARENT_PAGE_ID functions take no arguments. Fixes #74 * Add airtable_import_record_updated hook support to new importer * Add tests for updating pages, and publishing created pages --------- Co-authored-by: Jake Howard <[email protected]>
wagtail-nest · Dec 13, 2024 · 1096de3 · 1096de3
1 parent bc1604c
commit 1096de3
Show file tree

Hide file tree

Showing 19 changed files with 825 additions and 1,222 deletions.
diff --git a/README.md b/README.md
@@ -87,14 +87,11 @@ AIRTABLE_IMPORT_SETTINGS = {
         # Example:
         # 'PARENT_PAGE_ID': 3,
 
-        # If you choose option #1 (callable) or option #2 (path to a function)
-        # Your function needs to return an integer which will represent the Parent
-        # Page ID where all imported pages will be created as child pages.
-        # Callables and path-to-functions (option #1 and option #2 in the above docs)
-        # Take an `instance` kwarg as of v0.2.1. Example below:
+        # If you choose option #1 (callable) or option #2 (path to a function),
+        # the supplied function takes no arguments and returns the ID of the parent
+        # page where all imported pages will be created as child pages. For example:
         #   def custom_parent_page_id_function(instance=None):
-        #       if instance and isinstance(instance, Page):
-        #           return Page.objects.get(pk=instance.id).get_parent()
+        #       Page.objects.get(slug="imported-pages").pk
         'PARENT_PAGE_ID': 'path.to.function',
         # The `AUTO_PUBLISH_NEW_PAGES` setting will tell this package to either
         # Automatically publish a newly created page, or set to draft.

diff --git a/tests/fixtures/test.json b/tests/fixtures/test.json
@@ -51,7 +51,7 @@
     "model": "wagtailcore.page",
     "fields": {
         "title": "Home",
-        "numchild": 3,
+        "numchild": 0,
         "show_in_menus": false,
         "live": true,
         "depth": 2,

diff --git a/tests/migrations/0004_auto_20220526_2254.py → tests/migrations/0004_advert_publications.py b/tests/migrations/0004_auto_20220526_2254.py → tests/migrations/0004_advert_publications.py
@@ -20,6 +20,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='advert',
             name='publications',
-            field=models.ManyToManyField(blank=True, null=True, to='tests.Publication'),
+            field=models.ManyToManyField(blank=True, to='tests.Publication'),
         ),
     ]
diff --git a/tests/migrations/0005_simplepage_airtable_record_id.py b/tests/migrations/0005_simplepage_airtable_record_id.py
@@ -0,0 +1,18 @@
+# Generated by Django 5.0.9 on 2024-12-12 23:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("tests", "0004_advert_publications"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="simplepage",
+            name="airtable_record_id",
+            field=models.CharField(blank=True, db_index=True, max_length=35),
+        ),
+    ]
diff --git a/tests/mock_airtable.py b/tests/mock_airtable.py
@@ -0,0 +1,160 @@
+"""A mocked Airtable API wrapper."""
+from unittest import mock
+
+def get_mock_airtable():
+    """
+    Wrap it in a function, so it's pure
+    """
+
+    class MockAirtable(mock.Mock):
+        def get_iter(self):
+            return [self.get_all()]
+
+
+    MockAirtable.table_name = "app_airtable_advert_base_key"
+
+    MockAirtable.get = mock.MagicMock("get")
+    MockAirtable.get.return_value = {
+        "id": "recNewRecordId",
+        "fields": {
+            "title": "Red! It's the new blue!",
+            "description": "Red is a scientifically proven color that moves faster than all other colors.",
+            "external_link": "https://example.com/",
+            "is_active": True,
+            "rating": "1.5",
+            "long_description": "<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.</p>",
+            "points": 95,
+            "slug": "red-its-new-blue",
+        },
+    }
+
+    MockAirtable.insert = mock.MagicMock("insert")
+    MockAirtable.insert.return_value = {
+        "id": "recNewRecordId",
+        "fields": {
+            "title": "Red! It's the new blue!",
+            "description": "Red is a scientifically proven color that moves faster than all other colors.",
+            "external_link": "https://example.com/",
+            "is_active": True,
+            "rating": "1.5",
+            "long_description": "<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.</p>",
+            "points": 95,
+            "slug": "red-its-new-blue",
+        },
+    }
+
+    MockAirtable.update = mock.MagicMock("update")
+    MockAirtable.update.return_value = {
+        "id": "recNewRecordId",
+        "fields": {
+            "title": "Red! It's the new blue!",
+            "description": "Red is a scientifically proven color that moves faster than all other colors.",
+            "external_link": "https://example.com/",
+            "is_active": True,
+            "rating": "1.5",
+            "long_description": "<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.</p>",
+            "points": 95,
+            "slug": "red-its-new-blue",
+        },
+    }
+
+    MockAirtable.delete = mock.MagicMock("delete")
+    MockAirtable.delete.return_value = {"deleted": True, "record": "recNewRecordId"}
+
+    MockAirtable.search = mock.MagicMock("search")
+    MockAirtable.search.return_value = [
+        {
+            "id": "recNewRecordId",
+            "fields": {
+                "title": "Red! It's the new blue!",
+                "description": "Red is a scientifically proven color that moves faster than all other colors.",
+                "external_link": "https://example.com/",
+                "is_active": True,
+                "rating": "1.5",
+                "long_description": "<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.</p>",
+                "points": 95,
+                "slug": "red-its-new-blue",
+            },
+        },
+        {
+            "id": "Different record",
+            "fields": {
+                "title": "Not the used record.",
+                "description": "This is only used for multiple responses from MockAirtable",
+                "external_link": "https://example.com/",
+                "is_active": False,
+                "rating": "5.5",
+                "long_description": "",
+                "points": 1,
+                "slug": "not-the-used-record",
+            },
+        },
+    ]
+
+    MockAirtable.get_all = mock.MagicMock("get_all")
+    MockAirtable.get_all.return_value = [
+        {
+            "id": "recNewRecordId",
+            "fields": {
+                "title": "Red! It's the new blue!",
+                "description": "Red is a scientifically proven color that moves faster than all other colors.",
+                "external_link": "https://example.com/",
+                "is_active": True,
+                "rating": "1.5",
+                "long_description": "<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit. Veniam laboriosam consequatur saepe. Repellat itaque dolores neque, impedit reprehenderit eum culpa voluptates harum sapiente nesciunt ratione.</p>",
+                "points": 95,
+                "slug": "delete-me",
+                "publications": [
+                    {"title": "Record 1 publication 1"},
+                    {"title": "Record 1 publication 2"},
+                    {"title": "Record 1 publication 3"},
+                ]
+            },
+        },
+        {
+            "id": "Different record",
+            "fields": {
+                "title": "Not the used record.",
+                "description": "This is only used for multiple responses from MockAirtable",
+                "external_link": "https://example.com/",
+                "is_active": False,
+                "rating": "5.5",
+                "long_description": "",
+                "points": 1,
+                "slug": "not-the-used-record",
+            },
+        },
+        {
+            "id": "recRecordThree",
+            "fields": {
+                "title": "A third record.",
+                "description": "This is only used for multiple responses from MockAirtable",
+                "external_link": "https://example.com/",
+                "is_active": False,
+                "rating": "5.5",
+                "long_description": "",
+                "points": 1,
+                "slug": "record-3",
+            },
+        },
+        {
+            "id": "recRecordFour",
+            "fields": {
+                "title": "A fourth record.",
+                "description": "This is only used for multiple responses from MockAirtable",
+                "external_link": "https://example.com/",
+                "is_active": False,
+                "rating": "5.5",
+                "long_description": "",
+                "points": 1,
+                "slug": "record-4",
+                "publications": [
+                    {"title": "Record 4 publication 1"},
+                    {"title": "Record 4 publication 2"},
+                    {"title": "Record 4 publication 3"},
+                ]
+            },
+        },
+    ]
+
+    return MockAirtable
diff --git a/tests/models.py b/tests/models.py
@@ -7,9 +7,29 @@
 from wagtail_airtable.mixins import AirtableMixin, SnippetImportActionMixin
 
 
-class SimplePage(Page):
+def get_import_parent_page():
+    return Page.objects.get(slug="home").pk
+
+
+class SimplePage(AirtableMixin, Page):
     intro = models.TextField()
 
+    @classmethod
+    def map_import_fields(cls):
+        mappings = {
+            "title": "title",
+            "slug": "slug",
+            "intro": "intro",
+        }
+        return mappings
+
+    def get_export_fields(self):
+        return {
+            "title": self.title,
+            "slug": self.slug,
+            "intro": self.intro,
+        }
+
 
 class Publication(models.Model):
     title = models.CharField(max_length=30)
@@ -38,7 +58,7 @@ class Advert(AirtableMixin, models.Model):
     long_description = RichTextField(blank=True, null=True)
     points = models.IntegerField(null=True, blank=True)
     slug = models.SlugField(max_length=100, unique=True, editable=True)
-    publications = models.ManyToManyField(Publication, null=True, blank=True)
+    publications = models.ManyToManyField(Publication, blank=True)
 
     @classmethod
     def map_import_fields(cls):

diff --git a/tests/serializers.py b/tests/serializers.py
@@ -37,3 +37,9 @@ class AdvertSerializer(AirtableSerializer):
     title = serializers.CharField(max_length=255)
     external_link = serializers.URLField(required=False)
     publications = PublicationsObjectsSerializer(required=False)
+
+
+class SimplePageSerializer(AirtableSerializer):
+    title = serializers.CharField(max_length=255, required=True)
+    slug = serializers.CharField(max_length=100, required=True)
+    intro = serializers.CharField()
diff --git a/tests/settings.py b/tests/settings.py
@@ -132,6 +132,7 @@
         'AIRTABLE_TABLE_NAME': 'xxx',
         'AIRTABLE_UNIQUE_IDENTIFIER': 'slug',
         'AIRTABLE_SERIALIZER': 'tests.serializers.SimplePageSerializer',
+        'PARENT_PAGE_ID': 'tests.models.get_import_parent_page',
     },
     'tests.Advert': {
         'AIRTABLE_BASE_KEY': 'app_airtable_advert_base_key',