Add test suite (#217)

* Add test suite

.env_test

@@ -0,0 +1,243 @@
+COMPOSE_PROJECT_NAME=importer
+# See https://github.com/containers/podman/issues/13889
+# DOCKER_BUILDKIT=0
+DOCKER_ENV=production
+# See https://github.com/geosolutions-it/geonode-generic/issues/28
+# to see why we force API version to 1.24
+DOCKER_API_VERSION="1.24"
+BACKUPS_VOLUME_DRIVER=local
+
+C_FORCE_ROOT=1
+FORCE_REINIT=false
+INVOKE_LOG_STDOUT=true
+
+# LANGUAGE_CODE=it-it
+# LANGUAGES=(('en-us','English'),('it-it','Italiano'))
+
+DJANGO_SETTINGS_MODULE=geonode.settings
+GEONODE_INSTANCE_NAME=importer
+
+# #################
+# backend
+# #################
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+GEONODE_DATABASE=geonode
+GEONODE_DATABASE_USER=geonode
+GEONODE_DATABASE_PASSWORD=geonode
+GEONODE_GEODATABASE=geonode_data
+GEONODE_GEODATABASE_USER=geonode_data
+GEONODE_GEODATABASE_PASSWORD=geonode_data
+GEONODE_DATABASE_SCHEMA=public
+GEONODE_GEODATABASE_SCHEMA=public
+DATABASE_HOST=db
+DATABASE_PORT=5432
+DATABASE_URL=postgis://geonode:geonode@db:5432/geonode
+GEODATABASE_URL=postgis://geonode_data:geonode_data@db:5432/geonode_data
+GEONODE_DB_CONN_MAX_AGE=0
+GEONODE_DB_CONN_TOUT=5
+DEFAULT_BACKEND_DATASTORE=datastore
+BROKER_URL=amqp://guest:guest@rabbitmq:5672/
+CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler
+ASYNC_SIGNALS=False
+
+SITEURL=http://localhost:8000/
+
+ALLOWED_HOSTS="['django', 'localhost', '127.0.0.1']"
+
+# Data Uploader
+DEFAULT_BACKEND_UPLOADER=geonode.importer
+TIME_ENABLED=True
+MOSAIC_ENABLED=False
+HAYSTACK_SEARCH=False
+HAYSTACK_ENGINE_URL=http://elasticsearch:9200/
+HAYSTACK_ENGINE_INDEX_NAME=haystack
+HAYSTACK_SEARCH_RESULTS_PER_PAGE=200
+
+# #################
+# nginx
+# HTTPD Server
+# #################
+GEONODE_LB_HOST_IP=django
+GEONODE_LB_PORT=8000
+GEOSERVER_LB_HOST_IP=geoserver
+GEOSERVER_LB_PORT=8080
+NGINX_BASE_URL=http://localhost
+
+# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only)
+# port where the server can be reached on HTTPS
+HTTP_HOST=localhost
+HTTPS_HOST=
+POSTGRESQL_MAX_CONNECTIONS=100
+HTTP_PORT=8000
+HTTPS_PORT=443
+
+# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work
+# with an ip) and it must be reachable from the outside. This can be one of the following :
+# disabled : we do not get a certificate at all (a placeholder certificate will be used)
+# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates)
+# production : we get a normal certificate (default)
+LETSENCRYPT_MODE=disabled
+# LETSENCRYPT_MODE=staging
+# LETSENCRYPT_MODE=production
+
+RESOLVER=127.0.0.11
+
+# #################
+# geoserver
+# #################
+GEOSERVER_WEB_UI_LOCATION=http://localhost:8000/geoserver/
+GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/
+GEOSERVER_LOCATION=http://geoserver:8080/geoserver/
+GEOSERVER_ADMIN_USER=admin
+GEOSERVER_ADMIN_PASSWORD=geoserver
+
+OGC_REQUEST_TIMEOUT=5
+OGC_REQUEST_MAX_RETRIES=0
+OGC_REQUEST_BACKOFF_FACTOR=0.3
+OGC_REQUEST_POOL_MAXSIZE=10
+OGC_REQUEST_POOL_CONNECTIONS=10
+
+# #################
+# catalogue
+# #################
+CATALOGUE_ENGINE=geonode.catalogue.backends.pycsw_local
+CATALOGUE_URL=http://localhost:8000/catalogue/csw
+
+# Java Options & Memory
+ENABLE_JSONP=true
+outFormat=text/javascript
+GEOSERVER_JAVA_OPTS='-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://localhost/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine'
+
+# #################
+# Security
+# #################
+# Admin Settings
+#
+# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time
+# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT,
+# otherwise the invoke updateadmin task will be skipped and the current password already stored
+# in DB will honored.
+
+ADMIN_USERNAME=admin
+ADMIN_PASSWORD=admin
+ADMIN_EMAIL=admin@localhost
+
+# EMAIL Notifications
+EMAIL_ENABLE=False
+DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend
+DJANGO_EMAIL_HOST=localhost
+DJANGO_EMAIL_PORT=25
+DJANGO_EMAIL_HOST_USER=
+DJANGO_EMAIL_HOST_PASSWORD=
+DJANGO_EMAIL_USE_TLS=False
+DJANGO_EMAIL_USE_SSL=False
+DEFAULT_FROM_EMAIL='GeoNode <[email protected]>'
+
+# Session/Access Control
+LOCKDOWN_GEONODE=False
+X_FRAME_OPTIONS="SAMEORIGIN"
+SESSION_EXPIRED_CONTROL_ENABLED=True
+DEFAULT_ANONYMOUS_VIEW_PERMISSION=True
+DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True
+
+CORS_ALLOW_ALL_ORIGINS=True
+GEOSERVER_CORS_ENABLED=True
+GEOSERVER_CORS_ALLOWED_ORIGINS=*
+GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS
+GEOSERVER_CORS_ALLOWED_HEADERS=*
+
+# Users Registration
+ACCOUNT_OPEN_SIGNUP=True
+ACCOUNT_EMAIL_REQUIRED=True
+ACCOUNT_APPROVAL_REQUIRED=False
+ACCOUNT_CONFIRM_EMAIL_ON_GET=False
+ACCOUNT_EMAIL_VERIFICATION=none
+ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False
+ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False
+ACCOUNT_AUTHENTICATION_METHOD=username_email
+AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True
+
+# OAuth2
+OAUTH2_API_KEY=
+OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV
+OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3
+
+SOCIALACCOUNT_OIDC_PROVIDER_ENABLED=True
+SOCIALACCOUNT_PROVIDER=google
+
+# GeoNode APIs
+API_LOCKDOWN=False
+TASTYPIE_APIKEY=
+
+# #################
+# Production and
+# Monitoring
+# #################
+DEBUG=False
+
+SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a'
+
+STATIC_ROOT=/mnt/volumes/statics/static/
+MEDIA_ROOT=/mnt/volumes/statics/uploaded/
+GEOIP_PATH=/mnt/volumes/statics/geoip.db
+
+CACHE_BUSTING_STATIC_ENABLED=False
+
+MEMCACHED_ENABLED=False
+MEMCACHED_BACKEND=django.core.cache.backends.memcached.MemcachedCache
+MEMCACHED_LOCATION=memcached:11211
+MEMCACHED_LOCK_EXPIRE=3600
+MEMCACHED_LOCK_TIMEOUT=10
+#
+# Options for memcached binary, e.g. -vvv to log all requests and cache hits
+#
+MEMCACHED_OPTIONS=
+
+MAX_DOCUMENT_SIZE=200
+CLIENT_RESULTS_LIMIT=5
+API_LIMIT_PER_PAGE=1000
+
+# GIS Client
+GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore
+MAPBOX_ACCESS_TOKEN=
+BING_API_KEY=
+GOOGLE_API_KEY=
+
+# Monitoring
+MONITORING_ENABLED=False
+MONITORING_DATA_TTL=365
+USER_ANALYTICS_ENABLED=True
+USER_ANALYTICS_GZIP=True
+CENTRALIZED_DASHBOARD_ENABLED=False
+MONITORING_SERVICE_NAME=local-geonode
+MONITORING_HOST_NAME=geonode
+
+# Other Options/Contribs
+MODIFY_TOPICCATEGORY=True
+AVATAR_GRAVATAR_SSL=True
+EXIF_ENABLED=True
+CREATE_LAYER=True
+FAVORITE_ENABLED=True
+
+# Advanced Workflow
+RESOURCE_PUBLISHING=False
+ADMIN_MODERATE_UPLOADS=False
+
+# PostgreSQL
+POSTGRESQL_MAX_CONNECTIONS=200
+
+# Common containers restart policy
+RESTART_POLICY_CONDITION="on-failure"
+RESTART_POLICY_DELAY="5s"
+RESTART_POLICY_MAX_ATTEMPTS="3"
+RESTART_POLICY_WINDOW=120s
+
+DEFAULT_MAX_UPLOAD_SIZE=5368709120
+DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=100
+
+# Azure AD
+MICROSOFT_TENANT_ID=
+AZURE_CLIENT_ID=
+AZURE_SECRET_KEY=
+AZURE_KEY=

.github/workflows/runtests.yml

@@ -0,0 +1,25 @@
+name: Run test suite
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Start containers
+      run: docker-compose -f "docker-compose-test.yaml" up -d --build
+    - name: Run django entrypoint
+      run: docker exec django4importer /bin/sh -c "sh /usr/src/importer/entrypoint_test.sh"
+    - name: Run geonode-importer tests
+      run: docker exec django4importer /bin/sh -c "sh /usr/src/importer/runtest.sh"
+    - name: Stop containers
+      if: always()
+      run: docker-compose -f "docker-compose-test.yaml" down

Dockerfile

@@ -0,0 +1,12 @@
+FROM geonode/geonode-base:latest-ubuntu-22.04
+
+RUN git clone https://github.com/GeoNode/geonode.git /usr/src/geonode
+RUN mkdir -p /usr/src/importer
+
+RUN cd ..
+COPY . /usr/src/importer/
+WORKDIR /usr/src/importer
+
+RUN pip install -r /usr/src/geonode/requirements.txt
+RUN pip install --upgrade  -e /usr/src/importer/
+RUN pip install coverage

docker-compose-test.yaml

@@ -0,0 +1,104 @@
+version: '3.9'
+
+services:
+
+  # Our custom django application. It includes Geonode.
+  django:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: django4importer
+    healthcheck:
+      test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://django:8000/"
+      start_period: 60s
+      interval: 60s
+      timeout: 10s
+      retries: 2
+    environment:
+      - IS_CELERY=False
+    command: 'sleep infinity'
+    depends_on:
+      db:
+        condition: service_healthy
+    env_file:
+      - .env_test
+    volumes:
+      - statics:/mnt/volumes/statics
+      - geoserver-data-dir:/geoserver_data/data
+      - data:/data
+      - tmp:/tmp
+      - .:/usr/src/importer
+
+
+  # Geoserver backend
+  geoserver:
+    image: geonode/geoserver:2.23.0
+    container_name: geoserver4importer
+    healthcheck:
+      test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows"
+      start_period: 60s
+      interval: 20s
+      timeout: 10s
+      retries: 5
+    env_file:
+      - .env_test
+    ports:
+      - "8080:8080"
+    volumes:
+      - statics:/mnt/volumes/statics
+      - geoserver-data-dir:/geoserver_data/data
+      - data:/data
+      - tmp:/tmp
+    restart: unless-stopped
+    depends_on:
+      data-dir-conf:
+        condition: service_healthy
+
+  data-dir-conf:
+    image: geonode/geoserver_data:2.23.0
+    container_name: gsconf4importer
+    entrypoint: sleep infinity
+    volumes:
+      - geoserver-data-dir:/geoserver_data/data
+    restart: unless-stopped
+    healthcheck:
+      test: "ls -A '/geoserver_data/data' | wc -l"
+
+  # PostGIS database.
+  db:
+    # use geonode official postgis 15 image
+    image: geonode/postgis:15
+    command: postgres -c "max_connections=100"
+    container_name: db4importer
+    env_file:
+      - .env_test
+    volumes:
+      - dbdata:/var/lib/postgresql/data
+      - dbbackups:/pg_backups
+    restart: unless-stopped
+    healthcheck:
+      test: "pg_isready -d postgres -U postgres"
+    # uncomment to enable remote connections to postgres
+    ports:
+      - "5432:5432"
+
+
+volumes:
+  statics:
+    name: importer-statics
+  nginx-confd:
+    name: importer-nginxconfd
+  nginx-certificates:
+    name: importer-nginxcerts
+  geoserver-data-dir:
+    name: importer-gsdatadir
+  dbdata:
+    name: importer-dbdata
+  dbbackups:
+    name: importer-dbbackups
+  data:
+    name: importer-data
+  tmp:
+    name: importer-tmp
+  rabbitmq:
+    name: importer-rabbitmq