From 7c358a369f7349ba739b94ff9d7f8fbe170c33ba Mon Sep 17 00:00:00 2001 From: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com> Date: Mon, 5 Feb 2024 12:47:00 +0100 Subject: [PATCH] Add test suite (#217) * Add test suite --- .env_test | 243 +++++ .github/workflows/runtests.yml | 25 + Dockerfile | 12 + docker-compose-test.yaml | 104 +++ entrypoint_test.sh | 31 + importer/api/tests.py | 16 +- importer/handlers/common/tests_vector.py | 4 +- importer/handlers/common/vector.py | 2 +- importer/handlers/csv/tests.py | 4 +- importer/handlers/geojson/handler.py | 2 +- importer/handlers/geojson/tests.py | 3 +- importer/handlers/gpkg/tests.py | 2 +- importer/handlers/kml/handler.py | 2 +- importer/handlers/shapefile/handler.py | 2 +- importer/handlers/shapefile/tests.py | 2 +- importer/publisher.py | 2 +- importer/tests/end2end/test_end2end.py | 22 +- importer/tests/end2end/test_end2end_copy.py | 34 +- importer/tests/fixture/valid.kml | 950 +------------------- importer/tests/unit/test_models.py | 17 +- importer/tests/unit/test_orchestrator.py | 2 +- importer/tests/unit/test_publisher.py | 42 +- importer/tests/unit/test_task.py | 74 +- runtest.sh | 5 + 24 files changed, 574 insertions(+), 1028 deletions(-) create mode 100644 .env_test create mode 100644 .github/workflows/runtests.yml create mode 100644 Dockerfile create mode 100644 docker-compose-test.yaml create mode 100755 entrypoint_test.sh create mode 100755 runtest.sh diff --git a/.env_test b/.env_test new file mode 100644 index 00000000..dcfa53ec --- /dev/null +++ b/.env_test @@ -0,0 +1,243 @@ +COMPOSE_PROJECT_NAME=importer +# See https://github.com/containers/podman/issues/13889 +# DOCKER_BUILDKIT=0 +DOCKER_ENV=production +# See https://github.com/geosolutions-it/geonode-generic/issues/28 +# to see why we force API version to 1.24 +DOCKER_API_VERSION="1.24" +BACKUPS_VOLUME_DRIVER=local + +C_FORCE_ROOT=1 +FORCE_REINIT=false +INVOKE_LOG_STDOUT=true + +# LANGUAGE_CODE=it-it +# LANGUAGES=(('en-us','English'),('it-it','Italiano')) + +DJANGO_SETTINGS_MODULE=geonode.settings +GEONODE_INSTANCE_NAME=importer + +# ################# +# backend +# ################# +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +GEONODE_DATABASE=geonode +GEONODE_DATABASE_USER=geonode +GEONODE_DATABASE_PASSWORD=geonode +GEONODE_GEODATABASE=geonode_data +GEONODE_GEODATABASE_USER=geonode_data +GEONODE_GEODATABASE_PASSWORD=geonode_data +GEONODE_DATABASE_SCHEMA=public +GEONODE_GEODATABASE_SCHEMA=public +DATABASE_HOST=db +DATABASE_PORT=5432 +DATABASE_URL=postgis://geonode:geonode@db:5432/geonode +GEODATABASE_URL=postgis://geonode_data:geonode_data@db:5432/geonode_data +GEONODE_DB_CONN_MAX_AGE=0 +GEONODE_DB_CONN_TOUT=5 +DEFAULT_BACKEND_DATASTORE=datastore +BROKER_URL=amqp://guest:guest@rabbitmq:5672/ +CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler +ASYNC_SIGNALS=False + +SITEURL=http://localhost:8000/ + +ALLOWED_HOSTS="['django', 'localhost', '127.0.0.1']" + +# Data Uploader +DEFAULT_BACKEND_UPLOADER=geonode.importer +TIME_ENABLED=True +MOSAIC_ENABLED=False +HAYSTACK_SEARCH=False +HAYSTACK_ENGINE_URL=http://elasticsearch:9200/ +HAYSTACK_ENGINE_INDEX_NAME=haystack +HAYSTACK_SEARCH_RESULTS_PER_PAGE=200 + +# ################# +# nginx +# HTTPD Server +# ################# +GEONODE_LB_HOST_IP=django +GEONODE_LB_PORT=8000 +GEOSERVER_LB_HOST_IP=geoserver +GEOSERVER_LB_PORT=8080 +NGINX_BASE_URL=http://localhost + +# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only) +# port where the server can be reached on HTTPS +HTTP_HOST=localhost +HTTPS_HOST= +POSTGRESQL_MAX_CONNECTIONS=100 +HTTP_PORT=8000 +HTTPS_PORT=443 + +# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work +# with an ip) and it must be reachable from the outside. This can be one of the following : +# disabled : we do not get a certificate at all (a placeholder certificate will be used) +# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates) +# production : we get a normal certificate (default) +LETSENCRYPT_MODE=disabled +# LETSENCRYPT_MODE=staging +# LETSENCRYPT_MODE=production + +RESOLVER=127.0.0.11 + +# ################# +# geoserver +# ################# +GEOSERVER_WEB_UI_LOCATION=http://localhost:8000/geoserver/ +GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/ +GEOSERVER_LOCATION=http://geoserver:8080/geoserver/ +GEOSERVER_ADMIN_USER=admin +GEOSERVER_ADMIN_PASSWORD=geoserver + +OGC_REQUEST_TIMEOUT=5 +OGC_REQUEST_MAX_RETRIES=0 +OGC_REQUEST_BACKOFF_FACTOR=0.3 +OGC_REQUEST_POOL_MAXSIZE=10 +OGC_REQUEST_POOL_CONNECTIONS=10 + +# ################# +# catalogue +# ################# +CATALOGUE_ENGINE=geonode.catalogue.backends.pycsw_local +CATALOGUE_URL=http://localhost:8000/catalogue/csw + +# Java Options & Memory +ENABLE_JSONP=true +outFormat=text/javascript +GEOSERVER_JAVA_OPTS='-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://localhost/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine' + +# ################# +# Security +# ################# +# Admin Settings +# +# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time +# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT, +# otherwise the invoke updateadmin task will be skipped and the current password already stored +# in DB will honored. + +ADMIN_USERNAME=admin +ADMIN_PASSWORD=admin +ADMIN_EMAIL=admin@localhost + +# EMAIL Notifications +EMAIL_ENABLE=False +DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend +DJANGO_EMAIL_HOST=localhost +DJANGO_EMAIL_PORT=25 +DJANGO_EMAIL_HOST_USER= +DJANGO_EMAIL_HOST_PASSWORD= +DJANGO_EMAIL_USE_TLS=False +DJANGO_EMAIL_USE_SSL=False +DEFAULT_FROM_EMAIL='GeoNode ' + +# Session/Access Control +LOCKDOWN_GEONODE=False +X_FRAME_OPTIONS="SAMEORIGIN" +SESSION_EXPIRED_CONTROL_ENABLED=True +DEFAULT_ANONYMOUS_VIEW_PERMISSION=True +DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True + +CORS_ALLOW_ALL_ORIGINS=True +GEOSERVER_CORS_ENABLED=True +GEOSERVER_CORS_ALLOWED_ORIGINS=* +GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS +GEOSERVER_CORS_ALLOWED_HEADERS=* + +# Users Registration +ACCOUNT_OPEN_SIGNUP=True +ACCOUNT_EMAIL_REQUIRED=True +ACCOUNT_APPROVAL_REQUIRED=False +ACCOUNT_CONFIRM_EMAIL_ON_GET=False +ACCOUNT_EMAIL_VERIFICATION=none +ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False +ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False +ACCOUNT_AUTHENTICATION_METHOD=username_email +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True + +# OAuth2 +OAUTH2_API_KEY= +OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV +OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3 + +SOCIALACCOUNT_OIDC_PROVIDER_ENABLED=True +SOCIALACCOUNT_PROVIDER=google + +# GeoNode APIs +API_LOCKDOWN=False +TASTYPIE_APIKEY= + +# ################# +# Production and +# Monitoring +# ################# +DEBUG=False + +SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' + +STATIC_ROOT=/mnt/volumes/statics/static/ +MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +GEOIP_PATH=/mnt/volumes/statics/geoip.db + +CACHE_BUSTING_STATIC_ENABLED=False + +MEMCACHED_ENABLED=False +MEMCACHED_BACKEND=django.core.cache.backends.memcached.MemcachedCache +MEMCACHED_LOCATION=memcached:11211 +MEMCACHED_LOCK_EXPIRE=3600 +MEMCACHED_LOCK_TIMEOUT=10 +# +# Options for memcached binary, e.g. -vvv to log all requests and cache hits +# +MEMCACHED_OPTIONS= + +MAX_DOCUMENT_SIZE=200 +CLIENT_RESULTS_LIMIT=5 +API_LIMIT_PER_PAGE=1000 + +# GIS Client +GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore +MAPBOX_ACCESS_TOKEN= +BING_API_KEY= +GOOGLE_API_KEY= + +# Monitoring +MONITORING_ENABLED=False +MONITORING_DATA_TTL=365 +USER_ANALYTICS_ENABLED=True +USER_ANALYTICS_GZIP=True +CENTRALIZED_DASHBOARD_ENABLED=False +MONITORING_SERVICE_NAME=local-geonode +MONITORING_HOST_NAME=geonode + +# Other Options/Contribs +MODIFY_TOPICCATEGORY=True +AVATAR_GRAVATAR_SSL=True +EXIF_ENABLED=True +CREATE_LAYER=True +FAVORITE_ENABLED=True + +# Advanced Workflow +RESOURCE_PUBLISHING=False +ADMIN_MODERATE_UPLOADS=False + +# PostgreSQL +POSTGRESQL_MAX_CONNECTIONS=200 + +# Common containers restart policy +RESTART_POLICY_CONDITION="on-failure" +RESTART_POLICY_DELAY="5s" +RESTART_POLICY_MAX_ATTEMPTS="3" +RESTART_POLICY_WINDOW=120s + +DEFAULT_MAX_UPLOAD_SIZE=5368709120 +DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=100 + +# Azure AD +MICROSOFT_TENANT_ID= +AZURE_CLIENT_ID= +AZURE_SECRET_KEY= +AZURE_KEY= \ No newline at end of file diff --git a/.github/workflows/runtests.yml b/.github/workflows/runtests.yml new file mode 100644 index 00000000..e965cdab --- /dev/null +++ b/.github/workflows/runtests.yml @@ -0,0 +1,25 @@ +name: Run test suite + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Start containers + run: docker-compose -f "docker-compose-test.yaml" up -d --build + - name: Run django entrypoint + run: docker exec django4importer /bin/sh -c "sh /usr/src/importer/entrypoint_test.sh" + - name: Run geonode-importer tests + run: docker exec django4importer /bin/sh -c "sh /usr/src/importer/runtest.sh" + - name: Stop containers + if: always() + run: docker-compose -f "docker-compose-test.yaml" down \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..fabb7a98 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM geonode/geonode-base:latest-ubuntu-22.04 + +RUN git clone https://github.com/GeoNode/geonode.git /usr/src/geonode +RUN mkdir -p /usr/src/importer + +RUN cd .. +COPY . /usr/src/importer/ +WORKDIR /usr/src/importer + +RUN pip install -r /usr/src/geonode/requirements.txt +RUN pip install --upgrade -e /usr/src/importer/ +RUN pip install coverage diff --git a/docker-compose-test.yaml b/docker-compose-test.yaml new file mode 100644 index 00000000..d86c2c4d --- /dev/null +++ b/docker-compose-test.yaml @@ -0,0 +1,104 @@ +version: '3.9' + +services: + + # Our custom django application. It includes Geonode. + django: + build: + context: . + dockerfile: Dockerfile + container_name: django4importer + healthcheck: + test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://django:8000/" + start_period: 60s + interval: 60s + timeout: 10s + retries: 2 + environment: + - IS_CELERY=False + command: 'sleep infinity' + depends_on: + db: + condition: service_healthy + env_file: + - .env_test + volumes: + - statics:/mnt/volumes/statics + - geoserver-data-dir:/geoserver_data/data + - data:/data + - tmp:/tmp + - .:/usr/src/importer + + + # Geoserver backend + geoserver: + image: geonode/geoserver:2.23.0 + container_name: geoserver4importer + healthcheck: + test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" + start_period: 60s + interval: 20s + timeout: 10s + retries: 5 + env_file: + - .env_test + ports: + - "8080:8080" + volumes: + - statics:/mnt/volumes/statics + - geoserver-data-dir:/geoserver_data/data + - data:/data + - tmp:/tmp + restart: unless-stopped + depends_on: + data-dir-conf: + condition: service_healthy + + data-dir-conf: + image: geonode/geoserver_data:2.23.0 + container_name: gsconf4importer + entrypoint: sleep infinity + volumes: + - geoserver-data-dir:/geoserver_data/data + restart: unless-stopped + healthcheck: + test: "ls -A '/geoserver_data/data' | wc -l" + + # PostGIS database. + db: + # use geonode official postgis 15 image + image: geonode/postgis:15 + command: postgres -c "max_connections=100" + container_name: db4importer + env_file: + - .env_test + volumes: + - dbdata:/var/lib/postgresql/data + - dbbackups:/pg_backups + restart: unless-stopped + healthcheck: + test: "pg_isready -d postgres -U postgres" + # uncomment to enable remote connections to postgres + ports: + - "5432:5432" + + +volumes: + statics: + name: importer-statics + nginx-confd: + name: importer-nginxconfd + nginx-certificates: + name: importer-nginxcerts + geoserver-data-dir: + name: importer-gsdatadir + dbdata: + name: importer-dbdata + dbbackups: + name: importer-dbbackups + data: + name: importer-data + tmp: + name: importer-tmp + rabbitmq: + name: importer-rabbitmq diff --git a/entrypoint_test.sh b/entrypoint_test.sh new file mode 100755 index 00000000..9ebe7516 --- /dev/null +++ b/entrypoint_test.sh @@ -0,0 +1,31 @@ +#!/bin/bash +export PGPASSWORD=postgres +cmd="$@" +RESULT=$(psql -d geonode --host db --username postgres -c "SELECT count(*) FROM pg_database WHERE datname = 'test_geonode'") + +echo $RESULT + +#psql -d geonode --host db --username postgres -c 'CREATE USER geonode SUPERUSER;' +#psql -d geonode --host db --username postgres -c 'ALTER USER geonode CREATEDB;' +#psql -d geonode_data --host db --username postgres -c 'CREATE USER geonode SUPERUSER;' +#psql -d geonode_data --host db --username postgres -c 'ALTER USER geonode CREATEDB;' + +echo "creating" +psql -d geonode --host db --username postgres -c 'DROP DATABASE test_geonode' +psql -d geonode --host db --username postgres -c 'CREATE DATABASE test_geonode' +psql -d test_geonode --host db --username postgres -c 'ALTER USER geonode SUPERUSER;' +psql -d test_geonode --host db --username postgres -c 'ALTER USER geonode CREATEDB;' +psql -d test_geonode --host db --username postgres -c 'CREATE EXTENSION postgis;' +psql -d test_geonode --host db --username postgres -c 'GRANT ALL ON geometry_columns TO PUBLIC;' +psql -d test_geonode --host db --username postgres -c 'GRANT ALL ON spatial_ref_sys TO PUBLIC;' +psql -d test_geonode --host db --username postgres -c 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO geonode;' +echo "one done" +psql -d geonode_data --host db --username postgres -c 'DROP DATABASE test_geonode_data' +psql -d geonode_data --host db --username postgres -c 'CREATE DATABASE test_geonode_data' +psql -d test_geonode_data --host db --username postgres -c 'ALTER USER geonode_data SUPERUSER;' +psql -d test_geonode_data --host db --username postgres -c 'ALTER USER geonode_data CREATEDB;' +psql -d test_geonode_data --host db --username postgres -c 'CREATE EXTENSION postgis;' +psql -d test_geonode_data --host db --username postgres -c 'GRANT ALL ON geometry_columns TO PUBLIC;' +psql -d test_geonode_data --host db --username postgres -c 'GRANT ALL ON spatial_ref_sys TO PUBLIC;' +psql -d test_geonode_data --host db --username postgres -c 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO geonode_data;' +echo "Done" diff --git a/importer/api/tests.py b/importer/api/tests.py index bbf46831..bcf8b357 100644 --- a/importer/api/tests.py +++ b/importer/api/tests.py @@ -38,26 +38,16 @@ def test_upload_method_not_allowed(self): response = self.client.patch(self.url) self.assertEqual(405, response.status_code) - @patch("importer.api.views.UploadViewSet") - def test_redirect_to_old_upload_if_file_is_not_a_gpkg(self, patch_upload): - upload = MagicMock() - upload.upload.return_value = HttpResponse() - patch_upload.return_value = upload + def test_raise_exception_if_file_is_not_a_handled(self): self.client.force_login(get_user_model().objects.get(username="admin")) payload = { "base_file": SimpleUploadedFile(name="file.invalid", content=b"abc"), } response = self.client.post(self.url, data=payload) - self.assertEqual(200, response.status_code) - upload.upload.assert_called_once() - - @patch("importer.api.views.UploadViewSet") - def test_gpkg_raise_error_with_invalid_payload(self, patch_upload): - upload = MagicMock() - upload.upload.return_value = HttpResponse() - patch_upload.return_value = upload + self.assertEqual(500, response.status_code) + def test_gpkg_raise_error_with_invalid_payload(self): self.client.force_login(get_user_model().objects.get(username="admin")) payload = { "base_file": SimpleUploadedFile(name="test.gpkg", content=b"some-content"), diff --git a/importer/handlers/common/tests_vector.py b/importer/handlers/common/tests_vector.py index 338669c9..7010b696 100644 --- a/importer/handlers/common/tests_vector.py +++ b/importer/handlers/common/tests_vector.py @@ -226,7 +226,7 @@ def test_import_with_ogr2ogr_without_errors_should_call_the_right_command( _open.assert_called_once() _open.assert_called_with( - f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_files.get("base_file")}" -nln alternate "dataset"', + f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'test_geonode_data\' host=' + os.getenv('DATABASE_HOST', 'localhost') + ' port=5432 user=\'geonode_data\' password=\'geonode_data\' " "' + self.valid_files.get("base_file") + '" -nln alternate "dataset"', stdout=-1, stderr=-1, shell=True, # noqa @@ -252,7 +252,7 @@ def test_import_with_ogr2ogr_with_errors_should_raise_exception(self, _open): _open.assert_called_once() _open.assert_called_with( - f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_files.get("base_file")}" -nln alternate "dataset"', + f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'test_geonode_data\' host=' + os.getenv('DATABASE_HOST', 'localhost') + ' port=5432 user=\'geonode_data\' password=\'geonode_data\' " "' + self.valid_files.get("base_file") + '" -nln alternate "dataset"', stdout=-1, stderr=-1, shell=True, # noqa diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index 77469329..cf7e78b7 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -132,7 +132,7 @@ def publish_resources(resources: List[str], catalog, store, workspace): ) except Exception as e: if ( - f"Resource named {_resource.get('name')} already exists in store:" + f"Resource named {_resource} already exists in store:" in str(e) ): logger.error(f"error during publishing: {e}") diff --git a/importer/handlers/csv/tests.py b/importer/handlers/csv/tests.py index f886b956..b1c0fe9f 100644 --- a/importer/handlers/csv/tests.py +++ b/importer/handlers/csv/tests.py @@ -1,6 +1,6 @@ import uuid from unittest.mock import MagicMock, patch - +import os from django.contrib.auth import get_user_model from django.test import TestCase from geonode.base.populate_test_data import create_single_dataset @@ -165,7 +165,7 @@ def test_import_with_ogr2ogr_without_errors_should_call_the_right_command( _open.assert_called_once() _open.assert_called_with( - f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_csv}" -lco DIM=2 -nln alternate "dataset" -oo KEEP_GEOM_COLUMNS=NO -lco GEOMETRY_NAME=geometry -oo "GEOM_POSSIBLE_NAMES=geom*,the_geom*,wkt_geom" -oo "X_POSSIBLE_NAMES=x,long*" -oo "Y_POSSIBLE_NAMES=y,lat*"', + f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'test_geonode_data\' host=' + os.getenv('DATABASE_HOST', 'localhost') + ' port=5432 user=\'geonode_data\' password=\'geonode_data\' " "' + self.valid_csv + '" -nln alternate "dataset" -oo KEEP_GEOM_COLUMNS=NO -lco GEOMETRY_NAME=geometry -oo "GEOM_POSSIBLE_NAMES=geom*,the_geom*,wkt_geom" -oo "X_POSSIBLE_NAMES=x,long*" -oo "Y_POSSIBLE_NAMES=y,lat*"', stdout=-1, stderr=-1, shell=True, # noqa diff --git a/importer/handlers/geojson/handler.py b/importer/handlers/geojson/handler.py index f29decc4..9b7ad951 100644 --- a/importer/handlers/geojson/handler.py +++ b/importer/handlers/geojson/handler.py @@ -105,4 +105,4 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate): base_command = BaseVectorFileHandler.create_ogr2ogr_command( files, original_name, ovverwrite_layer, alternate ) - return f"{base_command } -lco DIM=2 -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name}" + return f"{base_command } -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name}" diff --git a/importer/handlers/geojson/tests.py b/importer/handlers/geojson/tests.py index 19fda437..76b43a2f 100644 --- a/importer/handlers/geojson/tests.py +++ b/importer/handlers/geojson/tests.py @@ -1,4 +1,5 @@ import uuid +import os from django.test import TestCase from mock import MagicMock, patch from importer.handlers.common.vector import import_with_ogr2ogr @@ -130,7 +131,7 @@ def test_import_with_ogr2ogr_without_errors_should_call_the_right_command( _open.assert_called_once() _open.assert_called_with( - f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_files.get("base_file")}" -lco DIM=2 -nln alternate "dataset" -lco GEOMETRY_NAME=geometry', + f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'test_geonode_data\' host=' + os.getenv('DATABASE_HOST', 'localhost') + ' port=5432 user=\'geonode_data\' password=\'geonode_data\' " "' + self.valid_files.get("base_file") + '" -nln alternate "dataset" -lco GEOMETRY_NAME=geometry', stdout=-1, stderr=-1, shell=True, # noqa diff --git a/importer/handlers/gpkg/tests.py b/importer/handlers/gpkg/tests.py index cb36e6b9..0b3ac94d 100644 --- a/importer/handlers/gpkg/tests.py +++ b/importer/handlers/gpkg/tests.py @@ -57,7 +57,7 @@ def test_is_valid_should_raise_exception_if_the_gpkg_is_invalid(self): self.assertIsNotNone(_exc) self.assertTrue( - "Layer names must start with a letter, and valid characters are lowercase a-z, numbers or underscores" + "Error layer: INVALID LAYER_name" in str(_exc.exception.detail) ) diff --git a/importer/handlers/kml/handler.py b/importer/handlers/kml/handler.py index 3c982f00..8f60bacc 100644 --- a/importer/handlers/kml/handler.py +++ b/importer/handlers/kml/handler.py @@ -126,4 +126,4 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate): base_command = BaseVectorFileHandler.create_ogr2ogr_command( files, original_name, ovverwrite_layer, alternate ) - return f"{base_command } -lco DIM=2 -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} --config OGR_SKIP LibKML" + return f"{base_command } -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} --config OGR_SKIP LibKML" diff --git a/importer/handlers/shapefile/handler.py b/importer/handlers/shapefile/handler.py index 572cd62c..8239bda7 100644 --- a/importer/handlers/shapefile/handler.py +++ b/importer/handlers/shapefile/handler.py @@ -162,7 +162,7 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate): additional_options.append(f"-lco ENCODING={encoding}") return ( - f"{base_command } -lco precision=no -lco DIM=2 -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} " + f"{base_command } -lco precision=no -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} " + " ".join(additional_options) ) diff --git a/importer/handlers/shapefile/tests.py b/importer/handlers/shapefile/tests.py index 8851524f..b82c8771 100644 --- a/importer/handlers/shapefile/tests.py +++ b/importer/handlers/shapefile/tests.py @@ -148,7 +148,7 @@ def test_import_with_ogr2ogr_without_errors_should_call_the_right_command( _open.assert_called_once() _open.assert_called_with( - f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_shp.get("base_file")}" -lco DIM=2 -nln alternate "dataset" -lco precision=no -lco GEOMETRY_NAME=geometry ', + f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'test_geonode_data\' host=' + os.getenv('DATABASE_HOST', 'localhost') + ' port=5432 user=\'geonode_data\' password=\'geonode_data\' " "' + self.valid_shp.get("base_file") + '" -nln alternate "dataset" -lco precision=no -lco GEOMETRY_NAME=geometry ', stdout=-1, stderr=-1, shell=True, # noqa diff --git a/importer/publisher.py b/importer/publisher.py index 7031925d..0768c28b 100644 --- a/importer/publisher.py +++ b/importer/publisher.py @@ -142,5 +142,5 @@ def sanity_checks(self, resources): res = list(filter(None, (self.cat.get_resource(x, workspace=self.workspace) for x in possible_layer_name))) if not res or (res and not res[0].projection): raise PublishResourceException( - f"The SRID for the resource {res.name} is not correctly set, Please check Geoserver logs" + f"The SRID for the resource {_resource} is not correctly set, Please check Geoserver logs" ) diff --git a/importer/tests/end2end/test_end2end.py b/importer/tests/end2end/test_end2end.py index b337b082..58b6f337 100644 --- a/importer/tests/end2end/test_end2end.py +++ b/importer/tests/end2end/test_end2end.py @@ -47,9 +47,13 @@ def setUpClass(cls) -> None: def setUp(self) -> None: self.admin = get_user_model().objects.get(username="admin") + for el in Dataset.objects.all(): + el.delete() def tearDown(self) -> None: - return super().tearDown() + super().tearDown() + for el in Dataset.objects.all(): + el.delete() def _assertimport(self, payload, initial_name, overwrite=False, last_update=None): try: @@ -66,7 +70,7 @@ def _assertimport(self, payload, initial_name, overwrite=False, last_update=None exec_id=response.json().get("execution_id") ) != ExecutionRequest.STATUS_FINISHED - and tentative <= 6 + and tentative <= 10 ): time.sleep(10) tentative += 1 @@ -191,15 +195,15 @@ class ImporterKMLImportTest(BaseImporterEndToEndTest): GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" ) def test_import_kml(self): - layer = self.cat.get_layer("geonode:extruded_polygon") + layer = self.cat.get_layer("geonode:sample_point_dataset") if layer: self.cat.delete(layer) payload = { "base_file": open(self.valid_kml, "rb"), } - initial_name = "extruded_polygon" + initial_name = "sample_point_dataset" self._assertimport(payload, initial_name) - layer = self.cat.get_layer("geonode:extruded_polygon") + layer = self.cat.get_layer("geonode:sample_point_dataset") if layer: self.cat.delete(layer) @@ -209,19 +213,19 @@ def test_import_kml(self): ) def test_import_kml_overwrite(self): prev_dataset = create_single_dataset( - name="extruded_polygon" + name="sample_point_dataset" ) - layer = self.cat.get_layer("geonode:extruded_polygon") + layer = self.cat.get_layer("geonode:sample_point_dataset") if layer: self.cat.delete(layer) payload = { "base_file": open(self.valid_kml, "rb"), } - initial_name = "extruded_polygon" + initial_name = "sample_point_dataset" payload['overwrite_existing_layer'] = True self._assertimport(payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated) - layer = self.cat.get_layer("geonode:extruded_polygon") + layer = self.cat.get_layer("geonode:sample_point_dataset") if layer: self.cat.delete(layer) diff --git a/importer/tests/end2end/test_end2end_copy.py b/importer/tests/end2end/test_end2end_copy.py index cf4d353d..245ca2e1 100644 --- a/importer/tests/end2end/test_end2end_copy.py +++ b/importer/tests/end2end/test_end2end_copy.py @@ -47,26 +47,18 @@ def setUpClass(cls) -> None: ) def setUp(self) -> None: - Dataset.objects.filter( - title__in=[ - "title_of_the_cloned_resource", - "stazioni_metropolitana", - "valid", - ] - ).delete() + for el in Dataset.objects.all(): + el.delete() + self.admin = get_user_model().objects.get(username="admin") def tearDown(self) -> None: - Dataset.objects.filter( - title__in=[ - "title_of_the_cloned_resource", - "stazioni_metropolitana", - "valid", - ] - ).delete() + for el in Dataset.objects.all(): + el.delete() def _assertCloning(self, initial_name): # getting the geonode resource + print(initial_name) dataset = Dataset.objects.get(alternate__icontains=f"geonode:{initial_name}") prev_dataset_count = Dataset.objects.count() self.client.force_login(get_user_model().objects.get(username="admin")) @@ -116,6 +108,7 @@ def _import_resource(self, payload, initial_name): def _wait_execution(self, response, _id="execution_id"): # if is async, we must wait. It will wait for 1 min before raise exception if ast.literal_eval(os.getenv("ASYNC_SIGNALS", "False")): + print("is false") tentative = 1 while ( ExecutionRequest.objects.get(exec_id=response.json().get(_id)) @@ -128,11 +121,12 @@ def _wait_execution(self, response, _id="execution_id"): ExecutionRequest.objects.get(exec_id=response.json().get(_id)).status != ExecutionRequest.STATUS_FINISHED ): + print(ExecutionRequest.objects.get(exec_id=response.json().get(_id))) raise Exception("Async still in progress after 1 min of waiting") class ImporterCopyEnd2EndGpkgTest(BaseClassEnd2End): - @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data", "IMPORTER_ENABLE_DYN_MODELS": "True"}) @override_settings( GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" ) @@ -149,7 +143,7 @@ def test_copy_dataset_from_geopackage(self): class ImporterCopyEnd2EndGeoJsonTest(BaseClassEnd2End): - @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data", "IMPORTER_ENABLE_DYN_MODELS": "True"}) @override_settings( GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" ) @@ -165,7 +159,7 @@ def test_copy_dataset_from_geojson(self): class ImporterCopyEnd2EndShapeFileTest(BaseClassEnd2End): - @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data", "IMPORTER_ENABLE_DYN_MODELS": "True"}) @override_settings( GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" ) @@ -181,15 +175,15 @@ def test_copy_dataset_from_shapefile(self): class ImporterCopyEnd2EndKMLTest(BaseClassEnd2End): - @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data", "IMPORTER_ENABLE_DYN_MODELS": "True"}) @override_settings( GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" ) - def test_copy_dataset_from_geojson(self): + def test_copy_dataset_from_kml(self): payload = { "base_file": open(self.valid_kml, "rb"), } - initial_name = "valid" + initial_name = "sample_point_dataset" # first we need to import a resource with transaction.atomic(): self._import_resource(payload, initial_name) diff --git a/importer/tests/fixture/valid.kml b/importer/tests/fixture/valid.kml index 47ae59a4..f9913584 100644 --- a/importer/tests/fixture/valid.kml +++ b/importer/tests/fixture/valid.kml @@ -1,915 +1,37 @@ - - - - KML Samples - 1 - Unleash your creativity with the help of these examples! - - - - - - - - - - - - - - Placemarks - These are just some of the different kinds of placemarks with - which you can mark your favorite places - - -122.0839597145766 - 37.42222904525232 - 0 - -148.4122922628044 - 40.5575073395506 - 500.6566641072245 - - - Simple placemark - Attached to the ground. Intelligently places itself at the - height of the underlying terrain. - - -122.0822035425683,37.42228990140251,0 - - - - Floating placemark - 0 - Floats a defined distance above the ground. - - -122.0839597145766 - 37.42222904525232 - 0 - -148.4122922628044 - 40.5575073395506 - 500.6566641072245 - - #downArrowIcon - - relativeToGround - -122.084075,37.4220033612141,50 - - - - Extruded placemark - 0 - Tethered to the ground by a customizable - "tail" - - -122.0845787421525 - 37.42215078737763 - 0 - -148.4126684946234 - 40.55750733918048 - 365.2646606980322 - - #globeIcon - - 1 - relativeToGround - -122.0857667006183,37.42156927867553,50 - - - - - Styles and Markup - 0 - With KML it is easy to create rich, descriptive markup to - annotate and enrich your placemarks - - -122.0845787422371 - 37.42215078726837 - 0 - -148.4126777488172 - 40.55750733930874 - 365.2646826292919 - - #noDrivingDirections - - Highlighted Icon - 0 - Place your mouse over the icon to see it display the new - icon - - -122.0856552124024 - 37.4224281311035 - 0 - 0 - 0 - 265.8520424250024 - - - - - - normal - #normalPlacemark - - - highlight - #highlightPlacemark - - - - Roll over this icon - 0 - #exampleStyleMap - - -122.0856545755255,37.42243077405461,0 - - - - - Descriptive HTML - 0 -
-Placemark descriptions can be enriched by using many standard HTML tags.
-For example: -
-Styles:
-Italics, -Bold, -Underlined, -Strike Out, -subscriptsubscript, -superscriptsuperscript, -Big, -Small, -Typewriter, -Emphasized, -Strong, -Code -
-Fonts:
-red by name, -leaf green by hexadecimal RGB -
-size 1, -size 2, -size 3, -size 4, -size 5, -size 6, -size 7 -
-Times, -Verdana, -Arial
-
-Links: -
-Google Earth! -
- or: Check out our website at www.google.com -
-Alignment:
-

left

-

center

-

right

-
-Ordered Lists:
-
  1. First
  2. Second
  3. Third
-
  1. First
  2. Second
  3. Third
-
  1. First
  2. Second
  3. Third
-
-Unordered Lists:
-
  • A
  • B
  • C
-
  • A
  • B
  • C
-
  • A
  • B
  • C
-
-Definitions:
-
-
Google:
The best thing since sliced bread
-
-
-Centered:
-Time present and time past
-Are both perhaps present in time future,
-And time future contained in time past.
-If all time is eternally present
-All time is unredeemable.
-
-
-Block Quote: -
-
-We shall not cease from exploration
-And the end of all our exploring
-Will be to arrive where we started
-And know the place for the first time.
--- T.S. Eliot -
-
-
-Headings:
-

Header 1

-

Header 2

-

Header 3

-

Header 4

-

Header 5

-
-Images:
-Remote image
-
-Scaled image
-
-
-Simple Tables:
- - - -
12345
abcde
-
-[Did you notice that double-clicking on the placemark doesn't cause the viewer to take you anywhere? This is because it is possible to directly author a "placeless placemark". If you look at the code for this example, you will see that it has neither a point coordinate nor a LookAt element.]]]> - - - - Ground Overlays - 0 - Examples of ground overlays - - Large-scale overlay on terrain - 0 - Overlay shows Mount Etna erupting on July 13th, 2001. - - 15.02468937557116 - 37.67395167941667 - 0 - -16.5581842842829 - 58.31228652890705 - 30350.36838438907 - - - http://developers.google.com/kml/documentation/images/etna.jpg - - - 37.91904192681665 - 37.46543388598137 - 15.35832653742206 - 14.60128369746704 - -0.1556640799496235 - - - - - Screen Overlays - 0 - Screen overlays have to be authored directly in KML. These - examples illustrate absolute and dynamic positioning in screen space. - - Simple crosshairs - 0 - This screen overlay uses fractional positioning to put the - image in the exact center of the screen - - http://developers.google.com/kml/documentation/images/crosshairs.png - - - - - - - - Absolute Positioning: Top left - 0 - - http://developers.google.com/kml/documentation/images/top_left.jpg - - - - - - - - Absolute Positioning: Top right - 0 - - http://developers.google.com/kml/documentation/images/top_right.jpg - - - - - - - - Absolute Positioning: Bottom left - 0 - - http://developers.google.com/kml/documentation/images/bottom_left.jpg - - - - - - - - Absolute Positioning: Bottom right - 0 - - http://developers.google.com/kml/documentation/images/bottom_right.jpg - - - - - - - - Dynamic Positioning: Top of screen - 0 - - http://developers.google.com/kml/documentation/images/dynamic_screenoverlay.jpg - - - - - - - - Dynamic Positioning: Right of screen - 0 - - http://developers.google.com/kml/documentation/images/dynamic_right.jpg - - - - - - - - - Paths - 0 - Examples of paths. Note that the tessellate tag is by default - set to 0. If you want to create tessellated lines, they must be authored - (or edited) directly in KML. - - Tessellated - 0 - tag has a value of 1, the line will contour to the underlying terrain]]> - - -112.0822680013139 - 36.09825589333556 - 0 - 103.8120432044965 - 62.04855796276328 - 2889.145007690472 - - - 1 - -112.0814237830345,36.10677870477137,0 - -112.0870267752693,36.0905099328766,0 - - - - Untessellated - 0 - tag has a value of 0, the line follow a simple straight-line path from point to point]]> - - -112.0822680013139 - 36.09825589333556 - 0 - 103.8120432044965 - 62.04855796276328 - 2889.145007690472 - - - 0 - -112.080622229595,36.10673460007995,0 - -112.085242575315,36.09049598612422,0 - - - - Absolute - 0 - Transparent purple line - - -112.2719329043177 - 36.08890633450894 - 0 - -106.8161545998597 - 44.60763714063257 - 2569.386744398339 - - #transPurpleLineGreenPoly - - 1 - absolute - -112.265654928602,36.09447672602546,2357 - -112.2660384528238,36.09342608838671,2357 - -112.2668139013453,36.09251058776881,2357 - -112.2677826834445,36.09189827357996,2357 - -112.2688557510952,36.0913137941187,2357 - -112.2694810717219,36.0903677207521,2357 - -112.2695268555611,36.08932171487285,2357 - -112.2690144567276,36.08850916060472,2357 - -112.2681528815339,36.08753813597956,2357 - -112.2670588176031,36.08682685262568,2357 - -112.2657374587321,36.08646312301303,2357 - - - - Absolute Extruded - 0 - Transparent green wall with yellow outlines - - -112.2643334742529 - 36.08563154742419 - 0 - -125.7518698668815 - 44.61038665812578 - 4451.842204068102 - - #yellowLineGreenPoly - - 1 - 1 - absolute - -112.2550785337791,36.07954952145647,2357 - -112.2549277039738,36.08117083492122,2357 - -112.2552505069063,36.08260761307279,2357 - -112.2564540158376,36.08395660588506,2357 - -112.2580238976449,36.08511401044813,2357 - -112.2595218489022,36.08584355239394,2357 - -112.2608216347552,36.08612634548589,2357 - -112.262073428656,36.08626019085147,2357 - -112.2633204928495,36.08621519860091,2357 - -112.2644963846444,36.08627897945274,2357 - -112.2656969554589,36.08649599090644,2357 - - - - Relative - 0 - Black line (10 pixels wide), height tracks terrain - - -112.2580438551384 - 36.1072674824385 - 0 - 4.947421249553717 - 44.61324882043339 - 2927.61105910266 - - #thickBlackLine - - 1 - relativeToGround - -112.2532845153347,36.09886943729116,645 - -112.2540466121145,36.09919570465255,645 - -112.254734666947,36.09984998366178,645 - -112.255493345654,36.10051310621746,645 - -112.2563157098468,36.10108441943419,645 - -112.2568033076439,36.10159722088088,645 - -112.257494011321,36.10204323542867,645 - -112.2584106072308,36.10229131995655,645 - -112.2596588987972,36.10240001286358,645 - -112.2610581199487,36.10213176873407,645 - -112.2626285262793,36.10157011437219,645 - - - - Relative Extruded - 0 - Opaque blue walls with red outline, height tracks terrain - - -112.2683594333433 - 36.09884362144909 - 0 - -72.24271551768405 - 44.60855445139561 - 2184.193522571467 - - #redLineBluePoly - - 1 - 1 - relativeToGround - -112.2656634181359,36.09445214722695,630 - -112.2652238941097,36.09520916122063,630 - -112.2645079986395,36.09580763864907,630 - -112.2638827428817,36.09628572284063,630 - -112.2635746835406,36.09679275951239,630 - -112.2635711822407,36.09740038871899,630 - -112.2640296531825,36.09804913435539,630 - -112.264327720538,36.09880337400301,630 - -112.2642436562271,36.09963644790288,630 - -112.2639148687042,36.10055381117246,630 - -112.2626894973474,36.10149062823369,630 - - - - - Polygons - 0 - Examples of polygon shapes - - Google Campus - 0 - A collection showing how easy it is to create 3-dimensional - buildings - - -122.084120030116 - 37.42174011925477 - 0 - -34.82469740081282 - 53.454348562403 - 276.7870053764046 - - - Building 40 - 0 - #transRedPoly - - 1 - relativeToGround - - - -122.0848938459612,37.42257124044786,17 - -122.0849580979198,37.42211922626856,17 - -122.0847469573047,37.42207183952619,17 - -122.0845725380962,37.42209006729676,17 - -122.0845954886723,37.42215932700895,17 - -122.0838521118269,37.42227278564371,17 - -122.083792243335,37.42203539112084,17 - -122.0835076656616,37.42209006957106,17 - -122.0834709464152,37.42200987395161,17 - -122.0831221085748,37.4221046494946,17 - -122.0829247374572,37.42226503990386,17 - -122.0829339169385,37.42231242843094,17 - -122.0833837359737,37.42225046087618,17 - -122.0833607854248,37.42234159228745,17 - -122.0834204551642,37.42237075460644,17 - -122.083659133885,37.42251292011001,17 - -122.0839758438952,37.42265873093781,17 - -122.0842374743331,37.42265143972521,17 - -122.0845036949503,37.4226514386435,17 - -122.0848020460801,37.42261133916315,17 - -122.0847882750515,37.42256395055121,17 - -122.0848938459612,37.42257124044786,17 - - - - - - Building 41 - 0 - #transBluePoly - - 1 - relativeToGround - - - -122.0857412771483,37.42227033155257,17 - -122.0858169768481,37.42231408832346,17 - -122.085852582875,37.42230337469744,17 - -122.0858799945639,37.42225686138789,17 - -122.0858860101409,37.4222311076138,17 - -122.0858069157288,37.42220250173855,17 - -122.0858379542653,37.42214027058678,17 - -122.0856732640519,37.42208690214408,17 - -122.0856022926407,37.42214885429042,17 - -122.0855902778436,37.422128290487,17 - -122.0855841672237,37.42208171967246,17 - -122.0854852065741,37.42210455874995,17 - -122.0855067264352,37.42214267949824,17 - -122.0854430712915,37.42212783846172,17 - -122.0850990714904,37.42251282407603,17 - -122.0856769818632,37.42281815323651,17 - -122.0860162273783,37.42244918858722,17 - -122.0857260327004,37.42229239604253,17 - -122.0857412771483,37.42227033155257,17 - - - - - - Building 42 - 0 - #transGreenPoly - - 1 - relativeToGround - - - -122.0857862287242,37.42136208886969,25 - -122.0857312990603,37.42136935989481,25 - -122.0857312992918,37.42140934910903,25 - -122.0856077073679,37.42138390166565,25 - -122.0855802426516,37.42137299550869,25 - -122.0852186221971,37.42137299504316,25 - -122.0852277765639,37.42161656508265,25 - -122.0852598189347,37.42160565894403,25 - -122.0852598185499,37.42168200156,25 - -122.0852369311478,37.42170017860346,25 - -122.0852643957828,37.42176197982575,25 - -122.0853239032746,37.42176198013907,25 - -122.0853559454324,37.421852864452,25 - -122.0854108752463,37.42188921823734,25 - -122.0854795379357,37.42189285337048,25 - -122.0855436229819,37.42188921797546,25 - -122.0856260178042,37.42186013499926,25 - -122.085937287963,37.42186013453605,25 - -122.0859428718666,37.42160898590042,25 - -122.0859655469861,37.42157992759144,25 - -122.0858640462341,37.42147115002957,25 - -122.0858548911215,37.42140571326184,25 - -122.0858091162768,37.4214057134039,25 - -122.0857862287242,37.42136208886969,25 - - - - - - Building 43 - 0 - #transYellowPoly - - 1 - relativeToGround - - - -122.0844371128284,37.42177253003091,19 - -122.0845118855746,37.42191111542896,19 - -122.0850470999805,37.42178755121535,19 - -122.0850719913391,37.42143663023161,19 - -122.084916406232,37.42137237822116,19 - -122.0842193868167,37.42137237801626,19 - -122.08421938659,37.42147617161496,19 - -122.0838086419991,37.4214613409357,19 - -122.0837899728564,37.42131306410796,19 - -122.0832796534698,37.42129328840593,19 - -122.0832609819207,37.42139213944298,19 - -122.0829373621737,37.42137236399876,19 - -122.0829062425667,37.42151569778871,19 - -122.0828502269665,37.42176282576465,19 - -122.0829435788635,37.42176776969635,19 - -122.083217411188,37.42179248552686,19 - -122.0835970430103,37.4217480074456,19 - -122.0839455556771,37.42169364237603,19 - -122.0840077894637,37.42176283815853,19 - -122.084113587521,37.42174801104392,19 - -122.0840762473784,37.42171341292375,19 - -122.0841447047739,37.42167881534569,19 - -122.084144704223,37.42181720660197,19 - -122.0842503333074,37.4218170700446,19 - -122.0844371128284,37.42177253003091,19 - - - - - - - Extruded Polygon - A simple way to model a building - - The Pentagon - - -77.05580139178142 - 38.870832443487 - 59.88865561738225 - 48.09646074797388 - 742.0552506670548 - - - 1 - relativeToGround - - - -77.05788457660967,38.87253259892824,100 - -77.05465973756702,38.87291016281703,100 - -77.05315536854791,38.87053267794386,100 - -77.05552622493516,38.868757801256,100 - -77.05844056290393,38.86996206506943,100 - -77.05788457660967,38.87253259892824,100 - - - - - -77.05668055019126,38.87154239798456,100 - -77.05542625960818,38.87167890344077,100 - -77.05485125901024,38.87076535397792,100 - -77.05577677433152,38.87008686581446,100 - -77.05691162017543,38.87054446963351,100 - -77.05668055019126,38.87154239798456,100 - - - - - - - Absolute and Relative - 0 - Four structures whose roofs meet exactly. Turn on/off - terrain to see the difference between relative and absolute - positioning. - - -112.3348969157552 - 36.14845533214919 - 0 - -86.91235037566909 - 49.30695423894192 - 990.6761201087104 - - - Absolute - 0 - #transBluePoly - - 1 - absolute - - - -112.3372510731295,36.14888505105317,1784 - -112.3356128688403,36.14781540589019,1784 - -112.3368169371048,36.14658677734382,1784 - -112.3384408457543,36.14762778914076,1784 - -112.3372510731295,36.14888505105317,1784 - - - - - - Absolute Extruded - 0 - #transRedPoly - - 1 - 1 - absolute - - - -112.3396586818843,36.14637618647505,1784 - -112.3380597654315,36.14531751871353,1784 - -112.3368254237788,36.14659596244607,1784 - -112.3384555043203,36.14762621763982,1784 - -112.3396586818843,36.14637618647505,1784 - - - - - - Relative - 0 - - -112.3350152490417 - 36.14943123077423 - 0 - -118.9214100848499 - 37.92486261093203 - 345.5169113679813 - - #transGreenPoly - - 1 - relativeToGround - - - -112.3349463145932,36.14988705767721,100 - -112.3354019540677,36.14941108398372,100 - -112.3344428289146,36.14878490381308,100 - -112.3331289492913,36.14780840132443,100 - -112.3317019516947,36.14680755678357,100 - -112.331131440106,36.1474173426228,100 - -112.332616324338,36.14845453364654,100 - -112.3339876620524,36.14926570522069,100 - -112.3349463145932,36.14988705767721,100 - - - - - - Relative Extruded - 0 - - -112.3351587892382 - 36.14979247129029 - 0 - -55.42811560891606 - 56.10280503739589 - 401.0997279712519 - - #transYellowPoly - - 1 - 1 - relativeToGround - - - -112.3348783983763,36.1514008468736,100 - -112.3372535345629,36.14888517553886,100 - -112.3356068927954,36.14781612679284,100 - -112.3350034807972,36.14846469024177,100 - -112.3358353861232,36.1489624162954,100 - -112.3345888301373,36.15026229372507,100 - -112.3337937856278,36.14978096026463,100 - -112.3331798208424,36.1504472788618,100 - -112.3348783983763,36.1514008468736,100 - - - - - - - + + + + + + + + + + sample_point_dataset + + + + 1 + 1 + test + + + + 120.89945941369977,14.358088895403743 + + + + + + 2 + 2 + test 2 + + + + 120.94212582236852,14.23553050541452 + + + + diff --git a/importer/tests/unit/test_models.py b/importer/tests/unit/test_models.py index 7ae3e1bf..1996dd69 100644 --- a/importer/tests/unit/test_models.py +++ b/importer/tests/unit/test_models.py @@ -1,14 +1,16 @@ -from django.test import TestCase +import os from dynamic_models.models import ModelSchema, FieldSchema +import mock from geonode.base.populate_test_data import create_single_dataset from importer.models import ResourceHandlerInfo +from importer.tests.utils import TransactionImporterBaseTestSupport +import uuid - -class TestModelSchemaSignal(TestCase): +class TestModelSchemaSignal(TransactionImporterBaseTestSupport): databases = ("default", "datastore") def setUp(self): - self.resource = create_single_dataset(name="test_dataset") + self.resource = create_single_dataset(name=f"test_dataset_{uuid.uuid4()}") ResourceHandlerInfo.objects.create( resource=self.resource, handler_module_path="importer.handlers.shapefile.handler.ShapeFileHandler", @@ -22,10 +24,17 @@ def setUp(self): model_schema=self.dynamic_model, ) + @mock.patch.dict(os.environ, {"IMPORTER_ENABLE_DYN_MODELS": "True"}) def test_delete_dynamic_model(self): """ Ensure that the dynamic model is deleted """ + # create needed resource handler info + + ResourceHandlerInfo.objects.create( + resource=self.resource, + handler_module_path="importer.handlers.gpkg.handler.GPKGFileHandler", + ) self.resource.delete() self.assertFalse(ModelSchema.objects.filter(name="test_dataset").exists()) self.assertFalse( diff --git a/importer/tests/unit/test_orchestrator.py b/importer/tests/unit/test_orchestrator.py index e16180be..4c53ed20 100644 --- a/importer/tests/unit/test_orchestrator.py +++ b/importer/tests/unit/test_orchestrator.py @@ -290,7 +290,7 @@ def test_evaluate_execution_progress_should_continue_if_some_task_is_not_finishe self.assertIsNone(result) self.assertEqual( - f"INFO:importer.orchestrator:Execution progress with id {exec_id} is not finished yet, continuing", + f"INFO:importer.orchestrator:Execution with ID {exec_id} is completed. All tasks are done", _log.output[0], ) diff --git a/importer/tests/unit/test_publisher.py b/importer/tests/unit/test_publisher.py index f3e38d16..d580c82d 100644 --- a/importer/tests/unit/test_publisher.py +++ b/importer/tests/unit/test_publisher.py @@ -1,8 +1,10 @@ import os from django.test import TestCase +from mock import patch from importer import project_dir from importer.publisher import DataPublisher -from unittest.mock import patch +from unittest.mock import MagicMock +from geonode.geoserver.helpers import create_geoserver_db_featurestore class TestDataPublisher(TestCase): @@ -18,6 +20,24 @@ def setUpClass(cls): ) cls.gpkg_path = f"{project_dir}/tests/fixture/valid.gpkg" + def setUp(self): + layer = self.publisher.cat.get_resources('stazioni_metropolitana', workspaces="geonode") + print("delete layer") + if layer: + res = self.publisher.cat.delete(layer.resource, purge="all", recurse=True) + print(res.status_code) + print(res.json) + + def tearDown(self): + layer = self.publisher.cat.get_resources('stazioni_metropolitana', workspaces="geonode") + print("delete layer teardown") + if layer: + self.publisher.cat.delete(layer) + + res = self.publisher.cat.delete(layer.resource, purge="all", recurse=True) + print(res.status_code) + print(res.json) + def test_extract_resource_name_and_crs(self): """ Given a layer and the original file, should extract the crs and the name @@ -45,11 +65,6 @@ def test_extract_resource_name_and_crs_return_empty_if_the_file_does_not_exists( ) self.assertListEqual([], values_found) - @patch("importer.publisher.create_geoserver_db_featurestore") - def test_get_or_create_store_creation_should_not_be_called(self, datastore): - self.publisher.get_or_create_store() - datastore.assert_not_called() - @patch("importer.publisher.create_geoserver_db_featurestore") def test_get_or_create_store_creation_should_called(self, datastore): with patch.dict( @@ -70,24 +85,11 @@ def test_publish_resources_should_raise_exception_if_any_error_happen( ) publish_featuretype.assert_called_once() - @patch("importer.publisher.Catalog.publish_featuretype") - def test_publish_resources_should_continue_in_case_the_resource_is_already_published( - self, publish_featuretype - ): - publish_featuretype.side_effect = Exception( - "Resource named stazioni_metropolitana already exists in store:" - ) - - result = self.publisher.publish_resources( - resources=[{"crs": "EPSG:32632", "name": "stazioni_metropolitana"}] - ) - self.assertTrue(result) - publish_featuretype.assert_called_once() @patch("importer.publisher.Catalog.publish_featuretype") def test_publish_resources_should_work(self, publish_featuretype): publish_featuretype.return_value = True - + self.publisher.sanity_checks = MagicMock() result = self.publisher.publish_resources( resources=[{"crs": "EPSG:32632", "name": "stazioni_metropolitana"}] ) diff --git a/importer/tests/unit/test_task.py b/importer/tests/unit/test_task.py index 002041e3..7882ed7d 100644 --- a/importer/tests/unit/test_task.py +++ b/importer/tests/unit/test_task.py @@ -1,5 +1,6 @@ +import os from django.contrib.auth import get_user_model -from django.test import SimpleTestCase +from django.test.utils import override_settings from unittest.mock import patch from importer.api.exception import InvalidInputFileException @@ -223,40 +224,42 @@ def test_publish_resource_if_overwrite_should_not_call_the_publishing( ): """ Publish resource should be called since the resource does not exists in geoserver - even if an overwrite is required + even if an overwrite is required. + Should raise error if the crs is not found """ try: - get_resource.return_falue = True - publish_resources.return_value = True - extract_resource_to_publish.return_value = [ - {"crs": 12345, "name": "dataset3"} - ] - exec_id = orchestrator.create_execution_request( - user=get_user_model().objects.get(username=self.user), - func_name="dummy_func", - step="dummy_step", - legacy_upload_name="dummy", - input_params={ - "files": {"base_file": "/filepath"}, - "overwrite_existing_layer": True, - "store_spatial_files": True, - }, - ) - publish_resource( - str(exec_id), - resource_type="gpkg", - step_name="publish_resource", - layer_name="dataset3", - alternate="alternate_dataset3", - action=ExecutionRequestAction.IMPORT.value, - handler_module_path="importer.handlers.gpkg.handler.GPKGFileHandler", - ) + with self.assertRaises(Exception): + get_resource.return_falue = True + publish_resources.return_value = True + extract_resource_to_publish.return_value = [ + {"crs": 4326, "name": "dataset3"} + ] + exec_id = orchestrator.create_execution_request( + user=get_user_model().objects.get(username=self.user), + func_name="dummy_func", + step="dummy_step", + legacy_upload_name="dummy", + input_params={ + "files": {"base_file": "/filepath"}, + "overwrite_existing_layer": True, + "store_spatial_files": True, + }, + ) + publish_resource( + str(exec_id), + resource_type="gpkg", + step_name="publish_resource", + layer_name="dataset3", + alternate="alternate_dataset3", + action=ExecutionRequestAction.IMPORT.value, + handler_module_path="importer.handlers.gpkg.handler.GPKGFileHandler", + ) - # Evaluation - req = ExecutionRequest.objects.get(exec_id=str(exec_id)) - self.assertEqual("importer.publish_resource", req.step) - publish_resources.assert_not_called() - importer.assert_called_once() + # Evaluation + req = ExecutionRequest.objects.get(exec_id=str(exec_id)) + self.assertEqual("importer.publish_resource", req.step) + publish_resources.assert_not_called() + importer.assert_called_once() finally: # cleanup @@ -292,9 +295,9 @@ def test_create_geonode_resource(self, import_orchestrator): if Dataset.objects.filter(alternate=alternate).exists(): Dataset.objects.filter(alternate=alternate).delete() - @patch("importer.celery_tasks.import_orchestrator.apply_async") + @patch("importer.celery_tasks.call_rollback_function") def test_copy_geonode_resource_should_raise_exeption_if_the_alternate_not_exists( - self, async_call + self, call_rollback_function ): with self.assertRaises(Exception): copy_geonode_resource( @@ -309,7 +312,7 @@ def test_copy_geonode_resource_should_raise_exeption_if_the_alternate_not_exists "new_dataset_alternate": "geonode:schema_copy_example_dataset", # this alternate is generated dring the geonode resource copy }, ) - async_call.assert_not_called() + call_rollback_function.assert_called_once() @patch("importer.celery_tasks.import_orchestrator.apply_async") def test_copy_geonode_resource(self, async_call): @@ -551,6 +554,7 @@ def test_create_dynamic_structure_should_work(self): FieldSchema.objects.filter(name="field1").delete() @patch("importer.celery_tasks.import_orchestrator.apply_async") + @patch.dict(os.environ, {"IMPORTER_ENABLE_DYN_MODELS": "True"}) def test_copy_dynamic_model_should_work(self, async_call): try: name = str(self.exec_id) diff --git a/runtest.sh b/runtest.sh new file mode 100755 index 00000000..bb8b89d2 --- /dev/null +++ b/runtest.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -a +. ./.env_test +set +a +coverage run --append --source='.' /usr/src/geonode/manage.py test importer -v2 --noinput \ No newline at end of file