Merged master into release

.env.example

@@ -14,7 +14,7 @@ DJANGO_ALLOWED_HOSTS="localhost 127.0.0.1 0.0.0.0 app nginx"
 
 SECRET_KEY=change_me
 
-LETSENCRYPT_EMAIL=[email protected]
+LETSENCRYPT_EMAIL="[email protected]"
 LETSENCRYPT_RSA_KEY_SIZE=4096
 # Set to 1 if you're testing your setup to avoid hitting request limits
 LETSENCRYPT_STAGING=1
@@ -56,8 +56,9 @@ HOST_POSTGRES_PORT=5433
 GEODB_HOST=geodb
 GEODB_PORT=5432
 GEODB_USER=postgres
-GEODB_PASSWORD=KUAa7h!G&wQEmkS3
+GEODB_PASSWORD="KUAa7h!G&wQEmkS3"
 GEODB_DB=postgres
+HOST_GEODB_PORT=5434
 
 # Sentry DSN. Missing value disables Sentry logging. Can be found on https://opengisch.sentry.io/settings/projects/qfieldcloud/keys/ .
 # DEFAULT: <NO VALUE>
@@ -85,7 +86,7 @@ EMAIL_USE_SSL=False
 EMAIL_PORT=25
 EMAIL_HOST_USER=user
 EMAIL_HOST_PASSWORD=password
-DEFAULT_FROM_EMAIL=webmaster@localhost
+DEFAULT_FROM_EMAIL="webmaster@localhost"
 
 QFIELDCLOUD_DEFAULT_NETWORK=qfieldcloud_default
 
@@ -111,6 +112,16 @@ QFIELDCLOUD_AUTH_TOKEN_EXPIRATION_HOURS=720
 # DEFAULT: "Europe/Zurich"
 QFIELDCLOUD_DEFAULT_TIME_ZONE="Europe/Zurich"
 
+# QFieldCloud QGIS image name to be used as worker by the `worker_wrapper`.
+# If empty value, a default name will be generated at build time, for example `qfieldcloud-qgis`.
+# DEFAULT: ""
+# QFIELDCLOUD_QGIS_IMAGE_NAME=""
+
+# QFieldCloud `libqfieldsync` volume path to be mounted by the `worker_wrapper` into `worker` containers.
+# If empty value or invalid value, the pip installed version defined in `requirements_libqfieldsync.txt` will be used.
+# DEFAULT: ""
+QFIELDCLOUD_LIBQFIELDSYNC_VOLUME_PATH=""
+
 # The Django development port. Not used in production.
 # DEFAULT: 8011
 DJANGO_DEV_PORT=8011

.github/workflows/test.yml

@@ -35,11 +35,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        django_apps:
+        django_app:
           - authentication
           - notifs
           - subscription
           - core
+          - __flaky__
     continue-on-error: true
     steps:
       - name: Checkout repo
@@ -73,17 +74,24 @@ jobs:
           docker compose run app python manage.py makemigrations --no-input --check
           docker compose run app python manage.py migrate
           docker compose run app python manage.py collectstatic
-      - name: Run unit and integration tests
+
+      - name: Run mandatory unit and integration tests
+        if: matrix.django_app != '__flaky__'
+        run: |
+          docker compose run app python manage.py test --keepdb -v2 --exclude-tag="flaky" qfieldcloud.${{ matrix.django_app }}
+
+      - name: Run flaky unit and integration tests
+        if: matrix.django_app == '__flaky__'
         run: |
-          docker compose run app python manage.py test --keepdb -v2 qfieldcloud.${{ matrix.django_apps }}
+          docker compose run app python manage.py test --keepdb -v2 --tag="flaky" qfieldcloud
 
       - name: "failure logs"
         if: failure()
         run: |
           docker compose logs
 
       - name: Post Google Chat message on failure
-        if: ${{ failure() }}
+        if: failure() && matrix.django_app != '__flaky__'
         uses: julb/action-post-googlechat-message@v1
         with:
           message: |

.gitignore

@@ -12,3 +12,4 @@ conf/nginx/certs/*
 conf/certbot/*
 Pipfile*
 **/site-packages
+docker-qgis/libqfieldsync

README.md

@@ -36,6 +36,9 @@ desire with a good editor:
     cp .env.example .env
     emacs .env
 
+
+Make sure the host's firewall allows port _8009_, required by the `minio` service. Failing to meet this requirement is likely to result in the service being unable to start.
+
 To build development images and run the containers:
 
     docker compose up -d --build
@@ -62,19 +65,67 @@ Now you can get started by adding the first user that would also be a super user
 
 ### Tests
 
-To run all the unit and functional tests (on a throwaway test
-database and a throwaway test storage directory):
+Rebuild the docker compose stack with the `docker-compose.override.test.yml` file added to the `COMPOSE_FILE` environment variable:
 
     export COMPOSE_FILE=docker-compose.yml:docker-compose.override.local.yml:docker-compose.override.test.yml
     # (Re-)build the app service to install necessary test utilities (requirements_test.txt)
     docker compose up -d --build
     docker compose run app python manage.py migrate
+    docker compose run app python manage.py collectstatic --noinput
+
+You can then run all the unit and functional tests:
+
     docker compose run app python manage.py test --keepdb
 
-To run only a test module (e.g. `test_permission.py`)
+To run only a test module (e.g. `test_permission.py`):
 
     docker compose run app python manage.py test --keepdb qfieldcloud.core.tests.test_permission
 
+To run a specific test:
+
+    docker compose run app python manage.py test --keepdb qfieldcloud.core.tests.test_permission.QfcTestCase.test_collaborator_project_takeover
+
+<details>
+<summary>
+Instructions to have a test instance running in parallel to a dev instance
+</summary>
+Create an <code>.env.test</code> file with the following variables that override the ones in <code>.env</code>:
+
+    ENVIRONMENT=test
+    QFIELDCLOUD_HOST=nginx
+    DJANGO_SETTINGS_MODULE=qfieldcloud.settings
+    STORAGE_ENDPOINT_URL=http://172.17.0.1:8109
+    MINIO_API_PORT=8109
+    MINIO_BROWSER_PORT=8110
+    WEB_HTTP_PORT=8101
+    WEB_HTTPS_PORT=8102
+    HOST_POSTGRES_PORT=8103
+    HOST_GEODB_PORT=8107
+    MEMCACHED_PORT=11212
+    QFIELDCLOUD_DEFAULT_NETWORK=qfieldcloud_test_default
+    QFIELDCLOUD_SUBSCRIPTION_MODEL=subscription.Subscription
+    DJANGO_DEV_PORT=8111
+    SMTP4DEV_WEB_PORT=8112
+    SMTP4DEV_SMTP_PORT=8125
+    SMTP4DEV_IMAP_PORT=8143
+    COMPOSE_PROJECT_NAME=qfieldcloud_test
+    COMPOSE_FILE=docker-compose.yml:docker-compose.override.local.yml:docker-compose.override.test.yml
+    DEBUG_DEBUGPY_APP_PORT=5781
+    DEBUG_DEBUGPY_WORKER_WRAPPER_PORT=5780
+    DEMGEN_PORT=8201
+
+Build the test docker compose stack:
+
+    docker compose --env-file .env --env-file .env.test up -d --build
+    docker compose --env-file .env --env-file .env.test run app python manage.py migrate
+    docker compose --env-file .env --env-file .env.test run app python manage.py collectstatic --noinput
+
+You can then launch the tests:
+
+    docker compose --env-file .env --env-file .env.test run app python manage.py test --keepdb
+
+</details>
+
 ### Debugging
 
 > This section gives examples for VSCode, please adapt to your IDE)

docker-app/manage.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python
 """Django's command-line utility for administrative tasks."""
-import os
 import sys
 
 
 def main():
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "qfieldcloud.settings")
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

docker-app/qfieldcloud/core/admin.py

@@ -606,6 +606,7 @@ class ProjectAdmin(QFieldCloudModelAdmin):
         "project_filename",
         "file_storage_bytes",
         "storage_keep_versions",
+        "packaging_offliner",
         "created_at",
         "updated_at",
         "data_last_updated_at",

docker-app/qfieldcloud/core/cron.py

@@ -1,5 +1,4 @@
 import logging
-import os
 from datetime import timedelta
 
 from constance import config
@@ -14,10 +13,6 @@
 
 logger = logging.getLogger(__name__)
 
-QGIS_CONTAINER_NAME = os.environ.get("QGIS_CONTAINER_NAME", None)
-
-assert QGIS_CONTAINER_NAME
-
 
 class DeleteExpiredInvitationsJob(CronJobBase):
     schedule = Schedule(run_every_mins=60)

docker-app/qfieldcloud/core/geodb_utils.py

@@ -1,6 +1,5 @@
-import os
-
 import psycopg2
+from django.conf import settings
 from psycopg2 import sql
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 
@@ -10,18 +9,18 @@ def __init__(self):
         pass
 
     def __enter__(self):
-        host = os.environ.get("GEODB_HOST")
-        port = os.environ.get("GEODB_PORT")
+        host = settings.GEODB_HOST
+        port = settings.GEODB_PORT
 
         # If geodb is running on the same machine we connect trough
         # the internal docker net
         if host == "geodb":
             port = 5432
 
         self.connection = psycopg2.connect(
-            dbname=os.environ.get("GEODB_DB"),
-            user=os.environ.get("GEODB_USER"),
-            password=os.environ.get("GEODB_PASSWORD"),
+            dbname=settings.GEODB_DB,
+            user=settings.GEODB_USER,
+            password=settings.GEODB_PASSWORD,
             host=host,
             port=port,
         )

docker-app/qfieldcloud/core/middleware/test.py

@@ -1,5 +1,4 @@
-import os
-
+from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import MiddlewareNotUsed
 
@@ -8,7 +7,7 @@ class TestMiddleware:
     def __init__(self, get_response):
         self.get_response = get_response
 
-        if os.environ.get("ENVIRONMENT") != "test":
+        if settings.ENVIRONMENT != "test":
             raise MiddlewareNotUsed()
 
     def __call__(self, request):

docker-app/qfieldcloud/core/migrations/0009_geodb.py

@@ -41,15 +41,13 @@ class Migration(migrations.Migration):
                 (
                     "hostname",
                     models.CharField(
-                        default=qfieldcloud.core.models.default_hostname,
+                        default=settings.GEODB_HOST,
                         max_length=255,
                     ),
                 ),
                 (
                     "port",
-                    models.PositiveIntegerField(
-                        default=qfieldcloud.core.models.default_port
-                    ),
+                    models.PositiveIntegerField(default=settings.GEODB_PORT),
                 ),
                 ("created_at", models.DateTimeField(auto_now_add=True)),
             ],

docker-app/qfieldcloud/core/migrations/0043_update_site_name.py

@@ -1,14 +1,13 @@
 # Generated by Django 3.2.7 on 2021-10-05 22:17
 
-import os
 
 from django.conf import settings
 from django.db import migrations
 
 
 def update_site_name(apps, schema_editor):
     site_model = apps.get_model("sites", "Site")
-    domain = os.environ.get("QFIELDCLOUD_HOST")
+    domain = settings.QFIELDCLOUD_HOST
     name = "QFieldCloud"
 
     site_model.objects.update_or_create(

docker-app/qfieldcloud/core/migrations/0073_project_packaging_offliner.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.2.23 on 2024-01-09 22:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0072_person_remaining_trial_organizations"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="project",
+            name="packaging_offliner",
+            field=models.CharField(
+                choices=[
+                    ("qgiscore", "QGIS Core Offline Editing (deprecated)"),
+                    ("pythonmini", "Optimized Packager"),
+                ],
+                default="qgiscore",
+                help_text='The Packaging Offliner packages data for offline use with QField. The new "Optimized Packager" should be preferred over the deprecated "QGIS Core Offline Editing" for new projects.',
+                max_length=100,
+                verbose_name="Packaging Offliner",
+            ),
+        ),
+    ]