Merged master into release

.env.example

@@ -71,9 +71,6 @@ SENTRY_SAMPLE_RATE=1
 # DEFAULT: dev
 SENTRY_RELEASE=dev
 
-REDIS_PASSWORD=change_me_with_a_very_loooooooooooong_password
-REDIS_PORT=6379
-
 # Memcached port. Exposed only in docker-compose.local.yml
 # DEFAULT: 11211
 MEMCACHED_PORT=11211

.github/workflows/build_and_push.yml

@@ -15,13 +15,13 @@ jobs:
       - name: Prepare
         id: prepare
         run: |
-          VERSION="latest"
-          TAG=""
+          TAG="latest"
+          COMMIT="commit-$(git rev-parse --short ${{ github.sha }})"
           if [[ $GITHUB_REF == refs/tags/* ]]; then
-            VERSION=${GITHUB_REF#refs/tags/v}
+            TAG=${GITHUB_REF#refs/tags/v}
           fi
-          TAG=${VERSION}
-          echo ::set-output name=tag::${TAG}
+          echo "docker_tag=${TAG}" >> $GITHUB_OUTPUT
+          echo "docker_commit=${COMMIT}" >> $GITHUB_OUTPUT
           git submodule update --init --recursive --depth 1
 
       - name: Set up Docker Buildx
@@ -61,7 +61,9 @@ jobs:
           context: ./docker-app
           file: ./docker-app/Dockerfile
           push: ${{ github.event_name != 'pull_request' }}
-          tags: opengisch/qfieldcloud-app:${{ steps.prepare.outputs.tag }}
+          tags: |
+            opengisch/qfieldcloud-app:${{ steps.prepare.outputs.docker_tag }}
+            opengisch/qfieldcloud-app:${{ steps.prepare.outputs.docker_commit }}
 
       - name: Docker Build and Push Worker
         id: docker_build_and_push_worker
@@ -72,26 +74,9 @@ jobs:
           context: ./docker-app
           file: ./docker-app/Dockerfile
           push: ${{ github.event_name != 'pull_request' }}
-          tags: opengisch/qfieldcloud-worker-wrapper:${{ steps.prepare.outputs.tag }}
-
-      # Redis
-      - name: Docker Test Redis
-        id: docker_test_redis
-        uses: docker/build-push-action@v2
-        with:
-          builder: ${{ steps.buildx.outputs.name }}
-          context: ./docker-redis
-          file: ./docker-redis/Dockerfile
-
-      - name: Docker Build and Push Redis
-        id: docker_build_and_push_redis
-        uses: docker/build-push-action@v2
-        with:
-          builder: ${{ steps.buildx.outputs.name }}
-          context: ./docker-redis
-          file: ./docker-redis/Dockerfile
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: opengisch/qfieldcloud-redis:${{ steps.prepare.outputs.tag }}
+          tags: |
+            opengisch/qfieldcloud-worker-wrapper:${{ steps.prepare.outputs.docker_tag }}
+            opengisch/qfieldcloud-worker-wrapper:${{ steps.prepare.outputs.docker_commit }}
 
       # QGIS
       - name: Docker Test QGIS
@@ -110,12 +95,6 @@ jobs:
           context: ./docker-qgis
           file: ./docker-qgis/Dockerfile
           push: ${{ github.event_name != 'pull_request' }}
-          tags: opengisch/qfieldcloud-qgis:${{ steps.prepare.outputs.tag }}
-
-      - name: Trigger deployment on private repository
-        uses: peter-evans/repository-dispatch@v1
-        with:
-          token: ${{ secrets.GIT_ACCESS_TOKEN }}
-          repository: opengisch/qfieldcloud-private
-          event-type: public_dispatch
-          client-payload: '{"version": "${{ steps.prepare.outputs.tag }}"}'
+          tags: |
+            opengisch/qfieldcloud-qgis:${{ steps.prepare.outputs.docker_tag }}
+            opengisch/qfieldcloud-qgis:${{ steps.prepare.outputs.docker_commit }}

.github/workflows/test.yml

@@ -1,7 +1,14 @@
 name: Test
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
+    branches:
+      - master
+      - release
   pull_request:
 
 jobs:
@@ -44,7 +51,7 @@ jobs:
         run: |
           ln -s docker-compose.override.local.yml docker-compose.override.yml
 
-      - name: Check env vars coniguration
+      - name: Check env vars configuration
         run: |
           scripts/check_envvars.sh
 
@@ -83,9 +90,9 @@ jobs:
             Failed job run for branch `${{ github.head_ref || github.ref_name }}`, check ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} .
           gchat_webhook_url: ${{ secrets.GOOGLE_CHAT_WEBHOOK_URL }}
 
-      - name: Setup tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 30
-        with:
-          limit-access-to-actor: true
+      # - name: Setup tmate session
+      #   if: ${{ failure() }}
+      #   uses: mxschmitt/action-tmate@v3
+      #   timeout-minutes: 30
+      #   with:
+      #     limit-access-to-actor: true

.pre-commit-config.yaml

@@ -51,6 +51,6 @@ repos:
     rev: 'v1.5.1'
     hooks:
       - id: mypy
-        additional_dependencies: [types-pytz, types-Deprecated, types-PyYAML, types-requests, types-redis, types-tabulate, types-jsonschema, django-stubs]
+        additional_dependencies: [types-pytz, types-Deprecated, types-PyYAML, types-requests, types-tabulate, types-jsonschema, django-stubs]
         pass_filenames: false
-        entry: bash -c 'mypy -p docker-qgis -p docker-app -p docker-redis "$@"' --
+        entry: bash -c 'mypy -p docker-qgis -p docker-app "$@"' --

README.md

@@ -245,7 +245,6 @@ Based on this example
 | nginx https   | 443   | WEB_HTTPS_PORT       | :white_check_mark: | :white_check_mark: | :white_check_mark: |
 | django http   | 8011  | DJANGO_DEV_PORT      | :white_check_mark: | :x:                | :x:                |
 | postgres      | 5433  | HOST_POSTGRES_PORT   | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| redis         | 6379  | REDIS_PORT           | :white_check_mark: | :white_check_mark: | :white_check_mark: |
 | memcached     | 11211 | MEMCACHED_PORT       | :white_check_mark: | :x:                | :x:                |
 | geodb         | 5432  | HOST_POSTGRES_PORT   | :white_check_mark: | :white_check_mark: | :x:                |
 | minio API     | 8009  | MINIO_API_PORT       | :white_check_mark: | :x:                | :x:                |

docker-app/qfieldcloud/authentication/authentication.py

@@ -4,11 +4,11 @@
 from django.utils import timezone
 from django.utils.translation import gettext as _
 from qfieldcloud.core.models import User
-from rest_framework import exceptions
 from rest_framework.authentication import (
     TokenAuthentication as DjangoRestFrameworkTokenAuthentication,
 )
 
+from ..core.exceptions import AuthenticationViaTokenFailedError
 from .models import AuthToken
 
 
@@ -54,13 +54,13 @@ def authenticate_credentials(self, key):
         try:
             token = model.objects.get(key=key)
         except model.DoesNotExist:
-            raise exceptions.AuthenticationFailed(_("Invalid token."))
+            raise AuthenticationViaTokenFailedError(_("Invalid token."))
 
         if not token.is_active:
-            raise exceptions.AuthenticationFailed(_("Token has expired."))
+            raise AuthenticationViaTokenFailedError(_("Token has expired."))
 
         if not token.user.is_active:
-            raise exceptions.AuthenticationFailed(_("User inactive or deleted."))
+            raise AuthenticationViaTokenFailedError(_("User inactive or deleted."))
 
         # update the token last used time
         # NOTE the UPDATE may be performed already on the `token = model.objects.get(key=key)`, but we lose "token has expired" exception.

docker-app/qfieldcloud/core/exceptions.py

@@ -48,6 +48,14 @@ class AuthenticationFailedError(QFieldCloudException):
     status_code = status.HTTP_401_UNAUTHORIZED
 
 
+class AuthenticationViaTokenFailedError(QFieldCloudException):
+    """Raised when QFieldCloud incoming request includes incorrect authentication token."""
+
+    code = "token_authentication_failed"
+    message = "Token authentication failed"
+    status_code = status.HTTP_401_UNAUTHORIZED
+
+
 class NotAuthenticatedError(QFieldCloudException):
     """Raised when QFieldCloud unauthenticated request fails the permission checks."""
 

docker-app/qfieldcloud/core/management/commands/createsuperuser.py

@@ -0,0 +1,14 @@
+from django.contrib.auth.management.commands.createsuperuser import (
+    Command as SuperUserCommand,
+)
+from qfieldcloud.core.models import Person
+
+
+class Command(SuperUserCommand):
+    """
+    We overwrite the django createsuperuser command because it uses the wrong User model
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.UserModel = Person

docker-app/qfieldcloud/core/management/commands/createuser.py

@@ -1,5 +1,5 @@
-from django.contrib.auth import get_user_model
 from django.core.management.base import BaseCommand
+from qfieldcloud.core.models import Person
 
 
 class Command(BaseCommand):
@@ -26,9 +26,8 @@ def handle(self, *args, **options):
         email = options.get("email")
         is_superuser = options.get("superuser")
         try:
-            User = get_user_model()
-            if not User.objects.filter(username=username).exists():
-                User.objects.create_user(
+            if not Person.objects.filter(username=username).exists():
+                Person.objects.create_user(
                     username=username,
                     email=email,
                     password=password,

docker-app/qfieldcloud/core/management/commands/status.py

@@ -9,11 +9,6 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         results = {}
 
-        results["redis"] = "ok"
-        # Check if redis is visible
-        if not utils.redis_is_running():
-            results["redis"] = "error"
-
         results["geodb"] = "ok"
         # Check geodb
         if not geodb_utils.geodb_is_running():

docker-app/qfieldcloud/core/tests/test_api.py

@@ -36,7 +36,6 @@ def setUp(self):
     def test_api_status(self):
         response = self.client.get("/api/v1/status/")
         self.assertTrue(status.is_success(response.status_code))
-        self.assertEqual(response.json()["redis"], "ok")
         self.assertEqual(response.json()["storage"], "ok")
         self.assertEqual(response.json()["geodb"], "ok")
 

docker-app/qfieldcloud/core/utils.py

@@ -14,7 +14,6 @@
 from botocore.errorfactory import ClientError
 from django.conf import settings
 from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
-from redis import Redis, exceptions
 
 logger = logging.getLogger(__name__)
 
@@ -85,18 +84,6 @@ def total_size(self) -> int:
         return sum(v.size for v in self.versions if v.size is not None)
 
 
-def redis_is_running() -> bool:
-    try:
-        connection = Redis(
-            "redis", password=os.environ.get("REDIS_PASSWORD"), port=6379
-        )
-        connection.set("foo", "bar")
-    except exceptions.ConnectionError:
-        return False
-
-    return True
-
-
 def get_s3_session() -> boto3.Session:
     """Get a new S3 Session instance using Django settings"""
 

docker-app/qfieldcloud/core/views/status_views.py

@@ -17,11 +17,6 @@ def get(self, request):
         # Try to get the status from the cache
         results = cache.get("status_results", {})
         if not results:
-            results["redis"] = "ok"
-            # Check if redis is visible
-            if not utils.redis_is_running():
-                results["redis"] = "error"
-
             results["geodb"] = "ok"
             # Check geodb
             if not geodb_utils.geodb_is_running():

docker-app/qfieldcloud/settings.py

@@ -68,7 +68,6 @@
 INSTALLED_APPS = [
     # django contrib
     "django.contrib.admin",
-    "django.contrib.auth",
     "django.contrib.contenttypes",
     "django.contrib.gis",
     "django.contrib.sessions",
@@ -96,6 +95,8 @@
     "auditlog",
     # Local
     "qfieldcloud.core",
+    # listed after core because we overwrite createsuperuser command
+    "django.contrib.auth",
     "qfieldcloud.subscription",
     "qfieldcloud.notifs",
     "qfieldcloud.authentication",

docker-app/qfieldcloud/subscription/models.py

@@ -776,6 +776,9 @@ def create_default_plan_subscription(
         else:
             created_by = account.user
 
+        if not isinstance(created_by, Person):
+            created_by = Person.objects.get(pk=created_by.pk)
+
         if active_since is None:
             active_since = timezone.now()
 
@@ -834,9 +837,14 @@ def create_subscription(
             # NOTE to get annotations, mostly `is_active`
             trial_subscription_obj = cls.objects.get(pk=trial_subscription.pk)
 
-            if created_by.remaining_trial_organizations > 0:
-                created_by.remaining_trial_organizations -= 1
-                created_by.save(update_fields=["remaining_trial_organizations"])
+            if (
+                account.user.is_organization
+                and account.user.organization_owner.remaining_trial_organizations > 0
+            ):
+                account.user.organization_owner.remaining_trial_organizations -= 1
+                account.user.organization_owner.save(
+                    update_fields=["remaining_trial_organizations"]
+                )
 
             # the trial plan should be the default plan
             regular_plan = Plan.objects.get(
@@ -854,10 +862,11 @@ def create_subscription(
             # NOTE in case the user had a custom amount set (e.g manually set by support) this will
             # be overwritten by a subscription plan change.
             # But taking care of this would add quite some complexity.
-            created_by.remaining_trial_organizations = (
-                regular_plan.max_trial_organizations
-            )
-            created_by.save(update_fields=["remaining_trial_organizations"])
+            if account.user.is_person:
+                account.user.remaining_trial_organizations = (
+                    regular_plan.max_trial_organizations
+                )
+                account.user.save(update_fields=["remaining_trial_organizations"])
 
         logger.info(f"Creating regular subscription from {regular_active_since}")
         regular_subscription = cls.objects.create(

docker-app/qfieldcloud/subscription/tests/test_subscription.py

@@ -731,33 +731,40 @@ def test_remaining_trial_organizations_is_set_and_decremented(
         user_plan.max_trial_organizations = 2
         user_plan.save(update_fields=["max_trial_organizations"])
         u1 = Person.objects.create(username="u1")
+        u2 = Person.objects.create(username="u2")
 
         # remaining_trial_organizations is set on create_subscription
         self.assertEqual(u1.remaining_trial_organizations, 2)
+        self.assertEqual(u2.remaining_trial_organizations, 2)
 
         trial_plan = Plan.objects.get(code="default_org")
         trial_plan.is_trial = True
         trial_plan.save(update_fields=["is_trial"])
 
-        def assert_create_org_decrements_count(
-            org_name, user, remaining_trial_organizations
-        ):
-            Organization.objects.create(
-                username=org_name, organization_owner=user, created_by=user
-            )
-            user.refresh_from_db()
-            self.assertEqual(
-                user.remaining_trial_organizations, remaining_trial_organizations
-            )
-
-        # remaining_trial_organizations is decremented when creating a trial organization
-        assert_create_org_decrements_count("org1", u1, 1)
+        # remaining_trial_organizations is decremented for owner when creating a trial organization
+        Organization.objects.create(
+            username="org2", organization_owner=u2, created_by=u1
+        )
+        u2.refresh_from_db()
+        self.assertEqual(u2.remaining_trial_organizations, 1)
+        u1.refresh_from_db()
+        self.assertEqual(u1.remaining_trial_organizations, 2)
 
         # remaining_trial_organizations is decremented down to 0
-        assert_create_org_decrements_count("org2", u1, 0)
+        Organization.objects.create(
+            username="org3", organization_owner=u2, created_by=u1
+        )
+        u2.refresh_from_db()
+        self.assertEqual(u2.remaining_trial_organizations, 0)
 
         # It doesn't directly prevent creating more trials, is just a counter that stays at 0
-        assert_create_org_decrements_count("org3", u1, 0)
+        Organization.objects.create(
+            username="org4", organization_owner=u2, created_by=u1
+        )
+        u2.refresh_from_db()
+        self.assertEqual(u2.remaining_trial_organizations, 0)
+
+        # NOTE changing ownership does not affect the `remaining_trial_organizations` and is not tested
 
     def test_project_lists_duplicates_if_multiple_subscriptions(self):
         u1 = Person.objects.create(username="u1")