Merge pull request #132 from liip/tests/test-against-latest-ckan-and-…

…ckanext-harvester Tests/test against latest ckan and ckanext harvester
liip · Nov 7, 2024 · 81b7b8e · 81b7b8e
2 parents 077f0e9 + 60ae239
commit 81b7b8e
Show file tree

Hide file tree

Showing 11 changed files with 36 additions and 51 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -22,7 +22,7 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        ckan-version: ["2.10"]
+        ckan-version: ["2.11"]
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
@@ -56,7 +56,7 @@ jobs:
           -v "${{ github.workspace }}/solr/stopwords_en.txt":"$SOLR_CONFIG_CKAN_DIR/stopwords_en.txt" \
           -v "${{ github.workspace }}/solr/stopwords_fr.txt":"$SOLR_CONFIG_CKAN_DIR/stopwords_fr.txt" \
           -v "${{ github.workspace }}/solr/stopwords_it.txt":"$SOLR_CONFIG_CKAN_DIR/stopwords_it.txt" \
-          ckan/ckan-solr:${{ matrix.ckan-version }}
+          ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
         docker start test_solr
     - name: Create ckan container
       run: |

diff --git a/bin/install_test_requirements.sh b/bin/install_test_requirements.sh
@@ -6,18 +6,14 @@ pip install --ignore-installed -r /__w/ckanext-switzerland/ckanext-switzerland/d
 pip install -e /__w/ckanext-switzerland/ckanext-switzerland/
 
 # Install ckanext dependencies
-pip install -e git+https://github.com/ckan/ckanext-dcat.git@v1.5.1#egg=ckanext-dcat
-pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v1.5.1/requirements.txt
+pip install -e git+https://github.com/ckan/ckanext-dcat.git#egg=ckanext-dcat
+pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/master/requirements.txt
 pip install -e git+https://gitlab.liip.ch/odp_oev_schweiz/ckanext-harvest.git#egg=ckanext-harvest
 pip install -r https://gitlab.liip.ch/odp_oev_schweiz/ckanext-harvest/-/raw/main/requirements.txt
-pip install -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming
+pip install -e git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming
 pip install -e git+https://github.com/ckan/ckanext-fluent.git#egg=ckanext-fluent
 pip install -r https://raw.githubusercontent.com/ckan/ckanext-fluent/master/requirements.txt
 
-# Replace default path to CKAN core config file with the one on the container
-sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' /__w/ckanext-switzerland/ckanext-switzerland/test.ini
-
 # Init db and re-enable required plugins
-ckan config-tool /__w/ckanext-switzerland/ckanext-switzerland/test.ini "ckan.plugins = "
 ckan -c /__w/ckanext-switzerland/ckanext-switzerland/test.ini db init
-ckan config-tool /__w/ckanext-switzerland/ckanext-switzerland/test.ini "ckan.plugins = ogdch ogdch_pkg ogdch_group ogdch_org harvest sbb_harvester timetable_harvester datastore fluent scheming_datasets scheming_groups scheming_organizations"
+ckan -c /__w/ckanext-switzerland/ckanext-switzerland/test.ini db pending-migrations --apply
diff --git a/ckanext/switzerland/tests/base_ftp_harvester_tests.py b/ckanext/switzerland/tests/base_ftp_harvester_tests.py
@@ -32,7 +32,6 @@ def run_harvester(
         ist_file=None,
         ftp_server=None,
     ):
-        data.harvest_user()
         self.user = data.user()
         self.organization = data.organization(self.user)
 

diff --git a/ckanext/switzerland/tests/data.py b/ckanext/switzerland/tests/data.py
@@ -80,10 +80,6 @@ def user():
     return factories.User()
 
 
-def harvest_user():
-    return factories.User(id="harvest", sysadmin=True)
-
-
 def organization(user):
     return factories.Organization(
         users=[{"name": user["id"], "capacity": "admin"}],

diff --git a/ckanext/switzerland/tests/fixtures/__init__.py b/ckanext/switzerland/tests/fixtures/__init__.py
@@ -2,13 +2,14 @@
 
 import pytest
 
-import ckanext.harvest.model as harvest_model
 from ckanext.harvest import queue
 
 
 @pytest.fixture
-def harvest_setup():
-    harvest_model.setup()
+def clean_db(reset_db, migrate_db_for):
+    reset_db()
+    migrate_db_for("harvest")
+    migrate_db_for("activity")
 
 
 @pytest.fixture

diff --git a/ckanext/switzerland/tests/test_infoplus.py b/ckanext/switzerland/tests/test_infoplus.py
@@ -54,7 +54,7 @@ def test_config(self):
             )
         )
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_simple(self):
         filesystem = self.get_filesystem(filename="FP2016_Jahresfahrplan.zip")
         MockFTPStorageAdapter.filesystem = filesystem

diff --git a/ckanext/switzerland/tests/test_ist_file.py b/ckanext/switzerland/tests/test_ist_file.py
@@ -32,7 +32,7 @@ class TestIstFileHarvester(BaseSBBHarvesterTests):
 
     harvester_class = SBBHarvester
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_simple(self):
         filesystem = self.get_filesystem(filename="ist_file.csv")
         path = os.path.join(data.environment, data.folder, "ist_file.csv")

diff --git a/ckanext/switzerland/tests/test_plugin.py b/ckanext/switzerland/tests/test_plugin.py
@@ -15,10 +15,7 @@
 log = logging.getLogger(__name__)
 
 
-@pytest.mark.ckan_config(
-    "ckan.plugins", "ogdch ogdch_pkg harvest fluent scheming_datasets activity"
-)
-@pytest.mark.usefixtures("clean_db", "clean_index", "with_plugins")
+@pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
 class TestOgdchPackagePlugin(object):
     @time_machine.travel(
         datetime.datetime(2022, 4, 20, 14, 15, 0, 0, ZoneInfo("UTC")), tick=False
@@ -45,14 +42,15 @@ def _create_dataset(self):
             "issued": "2022-04-18T12:00:00",
             "modified": "2022-04-18T12:30:00",
         }
+        user = data.user()
         dataset = data.dataset()
         resource = data.resource(dataset=dataset)
 
         dataset.update(**dataset_datetime_fields)
         resource.update(**resource_datetime_fields)
 
-        helpers.call_action("package_update", **dataset)
-        helpers.call_action("resource_create", **resource)
+        helpers.call_action("package_update", {"user": user["name"]}, **dataset)
+        helpers.call_action("resource_create", {"user": user["name"]}, **resource)
 
     def test_get_correct_datetime_format_from_api(self, app):
         self._create_dataset()

diff --git a/ckanext/switzerland/tests/test_sbb_harvester.py b/ckanext/switzerland/tests/test_sbb_harvester.py
@@ -40,7 +40,7 @@ class TestSBBHarvester(BaseSBBHarvesterTests):
 
     harvester_class = SBBHarvester
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_simple(self):
         MockFTPStorageAdapter.filesystem = self.get_filesystem()
         self.run_harvester(ftp_server="testserver")
@@ -58,7 +58,7 @@ def test_simple(self):
             relations=[],
         )
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_existing_dataset(self):
         data.dataset(slug="testslug-other-than-munge-name")
 
@@ -76,7 +76,7 @@ def test_existing_dataset(self):
         with self.assertRaises(NotFound):
             get_action("package_show")({}, {"id": munge_name(data.dataset_name)})
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_existing_resource(self):
         """
         Tests harvesting a new file which was not harvested before. Should create a new
@@ -106,7 +106,7 @@ def test_existing_resource(self):
         self.assertEqual(r1["description"]["de"], "AAAResource Desc")
         self.assertEqual(r2["description"]["de"], "AAAResource Desc")
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_existing_resource_same_filename(self):
         """
         Tests harvesting a new file which was not harvested before but manually uploaded
@@ -128,7 +128,7 @@ def test_existing_resource_same_filename(self):
 
         self.assertEqual(resource["description"]["de"], "AAAResource Desc")
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_skip_already_harvested_file(self):
         """
         When modified date of file is older than the last harvester run date, the file
@@ -145,7 +145,7 @@ def test_skip_already_harvested_file(self):
 
         self.assertEqual(len(package.resources), 1)
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_force_all(self):
         """
         When modified date of file is older than the last harvester run date, the file
@@ -173,7 +173,7 @@ def test_force_all(self):
             "The resource has not been harvested a second time",
         )
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_updated_file_before_last_harvester_run(self):
         """
         When modified date of file is older than the last harvester run date, the file
@@ -194,8 +194,7 @@ def test_updated_file_before_last_harvester_run(self):
         self.assertEqual(len(dataset["resources"]), 1)
         self.assertEqual(dataset["resources"][0]["identifier"], "Didok.csv")
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
-    @pytest.mark.ckan_config("ckan.site_url", "http://odp.test")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_update_version(self):
         filesystem = self.get_filesystem(filename="20160901.csv")
         MockFTPStorageAdapter.filesystem = filesystem
@@ -229,8 +228,7 @@ def test_update_version(self):
         self.assert_resource_data(package.resources[0].id, data.dataset_content_2)
         self.assert_resource_data(package.resources[1].id, data.dataset_content_1)
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
-    @pytest.mark.ckan_config("ckan.site_url", "http://odp.test")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_update_file_of_old_version(self):
         """
         initial state:
@@ -278,8 +276,7 @@ def test_update_file_of_old_version(self):
         self.assert_resource_data(package.resources[0].id, data.dataset_content_2)
         self.assert_resource_data(package.resources[1].id, data.dataset_content_3)
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
-    @pytest.mark.ckan_config("ckan.site_url", "http://odp.test")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_update_file_of_newest_version(self):
         """
         initial state:
@@ -325,8 +322,7 @@ def test_update_file_of_newest_version(self):
         self.assert_resource_data(package.resources[0].id, data.dataset_content_3)
         self.assert_resource_data(package.resources[1].id, data.dataset_content_1)
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
-    @pytest.mark.ckan_config("ckan.site_url", "http://odp.test")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_order_permalink_regex(self):
         filesystem = self.get_filesystem(filename="20160901.csv")
         MockFTPStorageAdapter.filesystem = filesystem
@@ -354,7 +350,7 @@ def test_order_permalink_regex(self):
             ),
         )
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     # cleanup tests
     def test_max_resources(self):
         filesystem = self.get_filesystem(filename="20160901.csv")
@@ -378,7 +374,7 @@ def test_max_resources(self):
         self.assertEqual(package.resources[1].extras["identifier"], "20160903.csv")
         self.assertEqual(package.resources[2].extras["identifier"], "20160902.csv")
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_resource_formats(self):
         filesystem = self.get_filesystem(filename="20160901.csv")
         MockFTPStorageAdapter.filesystem = filesystem
@@ -440,7 +436,7 @@ def test_resource_formats(self):
         # mimetype_inner is None, so CKAN doesn't save it on the resource
         self.assertNotIn("mimetype_inner", json_resource)
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_resource_formats_zip(self):
         currpath = os.path.dirname(os.path.realpath(__file__))
         src_path = os.path.join(currpath, "fixtures/zip/my.zip")
@@ -468,7 +464,7 @@ def test_resource_formats_zip(self):
         self.assertEqual(resource["mimetype"], "application/zip")
         self.assertEqual(resource["mimetype_inner"], "text/plain")
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_filter_regex(self):
         filesystem = self.get_filesystem(filename="File.zip")
         MockFTPStorageAdapter.filesystem = filesystem
@@ -515,7 +511,7 @@ def test_validate_regex_ok(self):
     @time_machine.travel(
         datetime(2022, 4, 20, 14, 15, 0, 0, ZoneInfo("UTC")), tick=False
     )
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_datetime_fields(self):
         """Test that all datetime fields for a new dataset are set to the current time
         in UTC.

diff --git a/ckanext/switzerland/tests/test_timetable_harvester.py b/ckanext/switzerland/tests/test_timetable_harvester.py
@@ -28,7 +28,7 @@ class TestTimetableHarvester(BaseSBBHarvesterTests):
 
     harvester_class = TimetableHarvester
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_simple(self):
         MockFTPStorageAdapter.filesystem = self.get_filesystem(
             filename="FP2016_Jahresfahrplan.zip"
@@ -54,7 +54,7 @@ def test_simple(self):
 
         self.assertEqual(len(dataset["resources"]), 1)
 
-    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup")
+    @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")
     def test_multi_year(self):
         filesystem = self.get_filesystem(filename="FP2016_Jahresfahrplan.zip")
         MockFTPStorageAdapter.filesystem = filesystem

diff --git a/test.ini b/test.ini
@@ -9,8 +9,7 @@ host = 0.0.0.0
 port = 5000
 
 [app:main]
-use = config:../../src/ckan/test-core.ini
-
+use = config:/srv/app/src/ckan/test-core.ini
 
 # custom config settings to be used when running tests
 # ----------------------------------------------------------------
@@ -36,7 +35,7 @@ ckan.ftp.testserver.port = 990
 ckan.ftp.testserver.remotedirectory = /
 ckan.ftp.testserver.localpath = /tmp/ftpharvest/tests/
 
-ckan.plugins = ogdch ogdch_pkg ogdch_group ogdch_org harvest sbb_harvester timetable_harvester datastore fluent scheming_datasets scheming_groups scheming_organizations
+ckan.plugins = ogdch ogdch_pkg ogdch_group ogdch_org harvest timetable_harvester sbb_harvester datastore fluent scheming_datasets scheming_groups scheming_organizations activity
 
 
 ckan.storage_path = /tmp/ckan_storage_path/