From 9858c2ddd7cd04a2bd184936dfba7feb6fb958a6 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Sat, 17 Sep 2022 19:32:37 +0300 Subject: [PATCH 01/16] Merge pull request #418 from opengisch/dependabot/pip/docker-app/oauthlib-3.2.1 --- docker-app/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-app/requirements.txt b/docker-app/requirements.txt index 926a4f0de..8f4494cb1 100644 --- a/docker-app/requirements.txt +++ b/docker-app/requirements.txt @@ -60,7 +60,7 @@ mergedeep==1.3.4 mkdocs==1.2.3 munch==2.5.0 mypy-boto3-s3==1.20.17 -oauthlib==3.1.1 +oauthlib==3.2.1 packaging==21.3 psycopg2-binary==2.8.6 pycparser==2.21 From 5a285f9183815fd640909c136afb73eb15b7e40d Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Tue, 11 Oct 2022 16:08:11 +0300 Subject: [PATCH 02/16] Merge pull request #425 from opengisch/custom-tag-for-logs --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2d187ef93..08decba1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,13 @@ version: "3.7" -x-logging: - &default-logging +x-logging: &default-logging driver: "json-file" options: max-size: "10m" max-file: "10" + tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}" services: - app: &default-django build: context: ./docker-app @@ -76,10 +75,11 @@ services: options: max-size: "1000m" max-file: "5" + tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}" labels: ofelia.enabled: "true" ofelia.job-exec.runcrons.no-overlap: "true" - ofelia.job-exec.runcrons.schedule: '@every 1m' + ofelia.job-exec.runcrons.schedule: "@every 1m" ofelia.job-exec.runcrons.command: python manage.py runcrons nginx: From bd73fe117581bb16610fa51caa5328dfad48f165 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Sun, 16 Oct 2022 04:46:38 +0300 Subject: [PATCH 03/16] Merge pull request #426 from opengisch/disable_i18n --- docker-app/qfieldcloud/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-app/qfieldcloud/settings.py b/docker-app/qfieldcloud/settings.py index b084c431e..5be24f774 100644 --- a/docker-app/qfieldcloud/settings.py +++ b/docker-app/qfieldcloud/settings.py @@ -191,7 +191,7 @@ TIME_ZONE = "Europe/Zurich" -USE_I18N = True +USE_I18N = False USE_L10N = True From 2c98f3d673b40001f58421c7ec72e21b4ee9788f Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Wed, 19 Oct 2022 12:59:16 +0300 Subject: [PATCH 04/16] Merge pull request #428 from opengisch/dependabot/pip/docker-app/django-3.2.16 --- docker-app/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-app/requirements.txt b/docker-app/requirements.txt index 8f4494cb1..2eae93739 100644 --- a/docker-app/requirements.txt +++ b/docker-app/requirements.txt @@ -17,7 +17,7 @@ coverage==5.3 cryptography==36.0.1 defusedxml==0.7.1 Deprecated==1.2.13 -Django==3.2.15 +Django==3.2.16 django-allauth==0.44.0 django-auditlog==1.0a1 django-axes==5.28.0 From 1f04aae7da6ba54bd5e6ffec51c33545ce757dfc Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Mon, 14 Nov 2022 11:49:59 +0200 Subject: [PATCH 05/16] Merge pull request #437 from opengisch/scale --- .env.example | 4 ++++ docker-compose.override.dev.yml | 2 +- docker-compose.override.local.yml | 3 ++- docker-compose.override.test.yml | 3 ++- docker-compose.yml | 3 ++- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 1b1e24d3c..8b8c0f8d4 100644 --- a/.env.example +++ b/.env.example @@ -77,6 +77,10 @@ QFIELDCLOUD_ADMIN_URI=admin/ # QFieldCloud URL used within the worker as configuration for qfieldcloud-sdk QFIELDCLOUD_WORKER_QFIELDCLOUD_URL=http://app:8000/api/v1/ +# number of parallel workers +# DEFAULT: 1 +QFIELDCLOUD_WORKER_REPLICAS=1 + # The Django development port. Not used in production. # DEFAULT: 8011 DJANGO_DEV_PORT=8011 diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index 0676ce03b..06edc228c 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: '3' services: diff --git a/docker-compose.override.local.yml b/docker-compose.override.local.yml index af1fd8757..d2679e9f9 100644 --- a/docker-compose.override.local.yml +++ b/docker-compose.override.local.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: '3' services: @@ -17,6 +17,7 @@ services: command: python3 manage.py runserver 0.0.0.0:8000 worker_wrapper: + scale: ${QFIELDCLOUD_WORKER_REPLICAS} build: args: - DEBUG_BUILD=1 diff --git a/docker-compose.override.test.yml b/docker-compose.override.test.yml index dcfb8df46..4bb8514d6 100644 --- a/docker-compose.override.test.yml +++ b/docker-compose.override.test.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: '3' services: @@ -21,6 +21,7 @@ services: command: python3 -m debugpy --listen 0.0.0.0:5681 manage.py dequeue ports: - 5681:5681 + scale: ${QFIELDCLOUD_WORKER_REPLICAS} db: environment: diff --git a/docker-compose.yml b/docker-compose.yml index 08decba1b..5056d5550 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.7" +version: "3.9" x-logging: &default-logging driver: "json-file" @@ -155,6 +155,7 @@ services: depends_on: - redis - app + scale: ${QFIELDCLOUD_WORKER_REPLICAS} ofelia: image: mcuadros/ofelia:v0.3.4 From 36fd6655cbc0f832e49c35ccfe4b2262120419c0 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Sat, 17 Dec 2022 14:59:26 +0200 Subject: [PATCH 06/16] Fix flake8 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index daf62e24b..ef4b40aca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: - id: black # Lint files - - repo: https://gitlab.com/pycqa/flake8 + - repo: https://github.com/pycqa/flake8 rev: "3.9.0" hooks: - id: flake8 From 3d9ab67c34d8a64842538b0bef6fd4c71848e83d Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Mon, 21 Nov 2022 11:27:46 +0200 Subject: [PATCH 07/16] Merge pull request #443 from opengisch/bump_qgis --- docker-qgis/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-qgis/Dockerfile b/docker-qgis/Dockerfile index 99f9420ea..d1ae073c7 100644 --- a/docker-qgis/Dockerfile +++ b/docker-qgis/Dockerfile @@ -1,4 +1,4 @@ -FROM qgis/qgis:final-3_26_1 +FROM qgis/qgis:final-3_28_1 RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ From e17259c62dc5028d5d5d2cfcfb13879e783a03c5 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Thu, 24 Nov 2022 05:13:06 +0200 Subject: [PATCH 08/16] Merge pull request #420 from miili/fix/empty-layer-id Closes https://github.com/opengisch/qfieldcloud/issues/415 --- .../qfieldcloud/core/tests/test_delta.py | 22 +++++++++++++++ ...yer_singledelta_empty_source_layer_id.json | 28 +++++++++++++++++++ docker-qgis/apply_deltas.py | 11 ++++++++ 3 files changed, 61 insertions(+) create mode 100644 docker-app/qfieldcloud/core/tests/testdata/delta/deltas/singlelayer_singledelta_empty_source_layer_id.json diff --git a/docker-app/qfieldcloud/core/tests/test_delta.py b/docker-app/qfieldcloud/core/tests/test_delta.py index 5a8965d15..fdec0f8ca 100644 --- a/docker-app/qfieldcloud/core/tests/test_delta.py +++ b/docker-app/qfieldcloud/core/tests/test_delta.py @@ -183,6 +183,28 @@ def test_push_apply_delta_file(self): features = list(layer) self.assertEqual(666, features[0]["properties"]["int"]) + def test_push_apply_delta_file_empty_source_layer_id(self): + self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key) + project = self.upload_project_files(self.project1) + + self.upload_and_check_deltas( + project=project, + delta_filename="singlelayer_singledelta_empty_source_layer_id.json", + token=self.token1.key, + final_values=[ + [ + "9311eb96-bff8-4d5b-ab36-c314a007cfcd", + "STATUS_APPLIED", + self.user1.username, + ] + ], + ) + + gpkg = io.BytesIO(self.get_file_contents(project, "testdata.gpkg")) + with fiona.open(gpkg, layer="points") as layer: + features = list(layer) + self.assertEqual(666, features[0]["properties"]["int"]) + def test_push_apply_delta_file_with_null_char(self): self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key) project = self.upload_project_files(self.project1) diff --git a/docker-app/qfieldcloud/core/tests/testdata/delta/deltas/singlelayer_singledelta_empty_source_layer_id.json b/docker-app/qfieldcloud/core/tests/testdata/delta/deltas/singlelayer_singledelta_empty_source_layer_id.json new file mode 100644 index 000000000..422f1b19e --- /dev/null +++ b/docker-app/qfieldcloud/core/tests/testdata/delta/deltas/singlelayer_singledelta_empty_source_layer_id.json @@ -0,0 +1,28 @@ +{ + "deltas": [ + { + "uuid": "9311eb96-bff8-4d5b-ab36-c314a007cfcd", + "clientId": "cd517e24-a520-4021-8850-e5af70e3a612", + "exportId": "f70c7286-fcec-4dbe-85b5-63d4735dac47", + "localPk": "1", + "sourcePk": "1", + "localLayerId": "points_897d5ed7_b810_4624_abe3_9f7c0a93d6a1", + "sourceLayerId": "", + "method": "patch", + "new": { + "attributes": { + "int": 666 + } + }, + "old": { + "attributes": { + "int": 1 + } + } + } + ], + "files": [], + "id": "6f109cd3-f44c-41db-b134-5f38468b9fda", + "project": "e02d02cc-af1b-414c-a14c-e2ed5dfee52f", + "version": "1.0" +} diff --git a/docker-qgis/apply_deltas.py b/docker-qgis/apply_deltas.py index 38a3be8c3..5a0f606d5 100755 --- a/docker-qgis/apply_deltas.py +++ b/docker-qgis/apply_deltas.py @@ -374,6 +374,17 @@ def delta_file_file_loader(args: DeltaOptions) -> Optional[DeltaFile]: obj["clientPks"], ) + # NOTE Sometimes QField does not fill the `sourceLayerId` field + # In recent QGIS versions, offline editing replaces the data source of the layers, so the layer ids do not change + # See https://github.com/opengisch/qfieldcloud/issues/415#issuecomment-1322922349 + for delta in delta_file.deltas: + if delta["sourceLayerId"] == "" and delta["localLayerId"] != "": + delta["sourceLayerId"] = delta["localLayerId"] + logger.warning( + "Patching project %s delta's empty sourceLayerId from localLayerId", + delta_file.project_id, + ) + return delta_file From 56a51dec34a34d626ddbbafdea317fb5193fcf6b Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Mon, 28 Nov 2022 16:00:24 +0200 Subject: [PATCH 09/16] Make constance not reset to default when the DB is down --- .../qfieldcloud/core/constance_backends.py | 31 +++++++++++++++++++ docker-app/qfieldcloud/settings.py | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 docker-app/qfieldcloud/core/constance_backends.py diff --git a/docker-app/qfieldcloud/core/constance_backends.py b/docker-app/qfieldcloud/core/constance_backends.py new file mode 100644 index 000000000..e0ea2df67 --- /dev/null +++ b/docker-app/qfieldcloud/core/constance_backends.py @@ -0,0 +1,31 @@ +from constance.backends.database import DatabaseBackend as BaseDatabaseBackend + + +class DatabaseBackend(BaseDatabaseBackend): + """ + Fix for https://github.com/jazzband/django-constance/issues/348 + Overrides the `get` method to remove silencing of database failures. + Such errors would otherwise result in unwanted resetting of parameters + to default values. + + Source: https://github.com/jazzband/django-constance/issues/348#issuecomment-676831598 + """ + + def get(self, key): + key = self.add_prefix(key) + if self._cache: + value = self._cache.get(key) + if value is None: + self.autofill() + value = self._cache.get(key) + else: + value = None + if value is None: + try: + value = self._model._default_manager.get(key=key).value + + if self._cache: + self._cache.add(key, value) + except self._model.DoesNotExist: # Only catch DoesNotExist exceptions here + pass + return value diff --git a/docker-app/qfieldcloud/settings.py b/docker-app/qfieldcloud/settings.py index 5be24f774..6193bff17 100644 --- a/docker-app/qfieldcloud/settings.py +++ b/docker-app/qfieldcloud/settings.py @@ -344,7 +344,7 @@ QFIELDCLOUD_ADMIN_URI = os.environ.get("QFIELDCLOUD_ADMIN_URI", "admin/") -CONSTANCE_BACKEND = "constance.backends.database.DatabaseBackend" +CONSTANCE_BACKEND = "qfieldcloud.core.constance_backends.DatabaseBackend" CONSTANCE_CONFIG = { "WORKER_TIMEOUT_S": ( 60, From 6dc07550570d9fd217cc0f2bb264b0e41365f03c Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Mon, 28 Nov 2022 21:49:50 +0200 Subject: [PATCH 10/16] Merge pull request #449 from opengisch/docker_compose_versions --- docker-compose.override.dev.yml | 2 +- docker-compose.override.local.yml | 2 +- docker-compose.override.test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index 06edc228c..0da9a32d6 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.9' services: diff --git a/docker-compose.override.local.yml b/docker-compose.override.local.yml index d2679e9f9..9becf6030 100644 --- a/docker-compose.override.local.yml +++ b/docker-compose.override.local.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.9' services: diff --git a/docker-compose.override.test.yml b/docker-compose.override.test.yml index 4bb8514d6..36f97afda 100644 --- a/docker-compose.override.test.yml +++ b/docker-compose.override.test.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.9' services: From ecdb26e4e07846cb3c7eeebc2b5bb1fd8b086960 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Thu, 1 Dec 2022 19:32:45 +0200 Subject: [PATCH 11/16] Merge pull request #451 from opengisch/optim --- docker-app/qfieldcloud/core/models.py | 16 +++++++++++----- docker-app/qfieldcloud/core/permissions_utils.py | 12 ++++++++++-- .../qfieldcloud/core/tests/test_queryset.py | 4 ++-- .../subscription/tests/test_organization.py | 4 ++-- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/docker-app/qfieldcloud/core/models.py b/docker-app/qfieldcloud/core/models.py index 3b27bfe32..ef94c8acb 100644 --- a/docker-app/qfieldcloud/core/models.py +++ b/docker-app/qfieldcloud/core/models.py @@ -75,9 +75,9 @@ def for_project(self, project: "Project", skip_invalid: bool): ) | ( Q(project_roles__project__owner__user_type=User.TYPE_ORGANIZATION) & Exists( - Organization.objects.of_user(OuterRef("project_roles__user")).filter( - id=OuterRef("project_roles__project__owner") - ) + Organization.objects.of_user(OuterRef("project_roles__user")) + .select_related(None) + .filter(id=OuterRef("project_roles__project__owner")) ) ) org_member = Case(When(org_member_condition, then=True), default=False) @@ -92,6 +92,7 @@ def for_project(self, project: "Project", skip_invalid: bool): user_type=User.TYPE_USER, project_roles__project=project, ) + .select_related("useraccount") .annotate( project_role=F("project_roles__name"), project_role_origin=F("project_roles__origin"), @@ -546,7 +547,7 @@ def get_queryset(self): return OrganizationQueryset(self.model, using=self._db) def of_user(self, user): - return self.get_queryset().of_user(user) + return self.get_queryset().select_related("useraccount").of_user(user) class Organization(User): @@ -818,7 +819,11 @@ def for_user(self, user: "User", skip_invalid: bool = False): org_member_condition = Q(owner__user_type=User.TYPE_USER) | ( Q(owner__user_type=User.TYPE_ORGANIZATION) - & Exists(Organization.objects.of_user(user).filter(id=OuterRef("owner"))) + & Exists( + Organization.objects.of_user(user) + .select_related(None) + .filter(id=OuterRef("owner")) + ) ) org_member = Case(When(org_member_condition, then=True), default=False) @@ -832,6 +837,7 @@ def for_user(self, user: "User", skip_invalid: bool = False): .filter( user_roles__user=user, ) + .select_related("owner") .annotate( user_role=F("user_roles__name"), user_role_origin=F("user_roles__origin"), diff --git a/docker-app/qfieldcloud/core/permissions_utils.py b/docker-app/qfieldcloud/core/permissions_utils.py index ac144e988..28a890529 100644 --- a/docker-app/qfieldcloud/core/permissions_utils.py +++ b/docker-app/qfieldcloud/core/permissions_utils.py @@ -44,11 +44,19 @@ class ExpectedPremiumUserError(CheckPermError): def _project_for_owner(user: QfcUser, project: Project, skip_invalid: bool): - return Project.objects.for_user(user, skip_invalid).filter(pk=project.pk) + return ( + Project.objects.for_user(user, skip_invalid) + .select_related(None) + .filter(pk=project.pk) + ) def _organization_of_owner(user: QfcUser, organization: Organization): - return Organization.objects.of_user(user).filter(pk=organization.pk) + return ( + Organization.objects.of_user(user) + .select_related(None) + .filter(pk=organization.pk) + ) def user_has_project_roles( diff --git a/docker-app/qfieldcloud/core/tests/test_queryset.py b/docker-app/qfieldcloud/core/tests/test_queryset.py index db23d5fd8..99cc524a4 100644 --- a/docker-app/qfieldcloud/core/tests/test_queryset.py +++ b/docker-app/qfieldcloud/core/tests/test_queryset.py @@ -159,10 +159,10 @@ def assertProjectRole( # Assert user does not have any role if role is None: with self.assertRaises(User.DoesNotExist): - User.objects.for_project(project).get(pk=user.pk) + User.objects.for_project(project).select_related(None).get(pk=user.pk) with self.assertRaises(Project.DoesNotExist): - Project.objects.for_user(user).get(pk=project.pk) + Project.objects.for_user(user).select_related(None).get(pk=project.pk) return diff --git a/docker-app/qfieldcloud/subscription/tests/test_organization.py b/docker-app/qfieldcloud/subscription/tests/test_organization.py index 4767c43a1..6a1afd45f 100644 --- a/docker-app/qfieldcloud/subscription/tests/test_organization.py +++ b/docker-app/qfieldcloud/subscription/tests/test_organization.py @@ -43,10 +43,10 @@ def assertProjectRole( # Assert user does not have any role if role is None: with self.assertRaises(User.DoesNotExist): - User.objects.for_project(project).get(pk=user.pk) + User.objects.for_project(project).select_related(None).get(pk=user.pk) with self.assertRaises(Project.DoesNotExist): - Project.objects.for_user(user).get(pk=project.pk) + Project.objects.for_user(user).select_related(None).get(pk=project.pk) return From 594d5d80a280b61f2537b2c9e3af74ec93a48c0f Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Mon, 5 Dec 2022 20:41:46 +0200 Subject: [PATCH 12/16] Merge pull request #450 from opengisch/admin_fixes --- .github/workflows/test.yml | 3 +- .../qfieldcloud/authentication/admin.py | 20 ++ docker-app/qfieldcloud/core/admin.py | 229 +++++++----------- .../qfieldcloud/core/tests/test_admin.py | 114 +++++++++ docker-app/qfieldcloud/notifs/admin.py | 38 ++- 5 files changed, 256 insertions(+), 148 deletions(-) create mode 100644 docker-app/qfieldcloud/authentication/admin.py create mode 100644 docker-app/qfieldcloud/core/tests/test_admin.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4044b434b..5a8593905 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,9 +35,10 @@ jobs: - name: Build and run docker containers run: | docker-compose up -d --build - - name: Migrate + - name: Initial manage.py commands run: | docker-compose run app python manage.py migrate + docker-compose run app python manage.py collectstatic - name: Run unit and integration tests run: | docker-compose run app python manage.py test --keepdb -v2 qfieldcloud diff --git a/docker-app/qfieldcloud/authentication/admin.py b/docker-app/qfieldcloud/authentication/admin.py new file mode 100644 index 000000000..d2d5ea5f7 --- /dev/null +++ b/docker-app/qfieldcloud/authentication/admin.py @@ -0,0 +1,20 @@ +from django.contrib import admin +from django.contrib.admin import register + +from .models import AuthToken + + +@register(AuthToken) +class AuthTokenAdmin(admin.ModelAdmin): + list_display = ("user", "created_at", "expires_at", "last_used_at", "client_type") + readonly_fields = ( + "key", + "user", + "created_at", + "last_used_at", + "client_type", + "user_agent", + ) + list_filter = ("created_at", "last_used_at", "expires_at") + + search_fields = ("user__username__iexact", "client_type", "key__startswith") diff --git a/docker-app/qfieldcloud/core/admin.py b/docker-app/qfieldcloud/core/admin.py index e83116a8a..2e501e699 100644 --- a/docker-app/qfieldcloud/core/admin.py +++ b/docker-app/qfieldcloud/core/admin.py @@ -12,16 +12,17 @@ from django.shortcuts import resolve_url from django.utils.html import escape, format_html from django.utils.safestring import SafeText +from invitations.admin import InvitationAdmin as InvitationAdminBase +from invitations.utils import get_invitation_model from qfieldcloud.core import exceptions from qfieldcloud.core.models import ( ApplyJob, ApplyJobDelta, Delta, Geodb, + Job, Organization, OrganizationMember, - PackageJob, - ProcessProjectfileJob, Project, ProjectCollaborator, Team, @@ -30,6 +31,34 @@ UserAccount, ) from qfieldcloud.core.utils2 import jobs +from rest_framework.authtoken.models import TokenProxy + +Invitation = get_invitation_model() + + +def admin_urlname_by_obj(value, arg): + if isinstance(value, User): + if value.is_user: + return "admin:core_user_%s" % (arg) + elif value.is_organization: + return "admin:core_organization_%s" % (arg) + elif value.is_team: + return "admin:core_team_%s" % (arg) + else: + raise NotImplementedError("Unknown user type!") + elif isinstance(value, Job): + return "admin:core_job_%s" % (arg) + else: + return admin_urlname(value._meta, arg) + + +# Unregister admins from other Django apps +admin.site.unregister(Invitation) +admin.site.unregister(TokenProxy) +admin.site.unregister(Group) +admin.site.unregister(SocialAccount) +admin.site.unregister(SocialApp) +admin.site.unregister(SocialToken) class PrettyJSONWidget(widgets.Textarea): @@ -62,7 +91,7 @@ def search_parser( def model_admin_url(obj, name: str = None) -> str: - url = resolve_url(admin_urlname(obj._meta, SafeText("change")), obj.pk) + url = resolve_url(admin_urlname_by_obj(obj, SafeText("change")), obj.pk) return format_html('{}', url, name or str(obj)) @@ -197,6 +226,7 @@ class UserAdmin(admin.ModelAdmin): "is_staff", "is_active", "date_joined", + "last_login", "user_type", "useraccount", ) @@ -369,33 +399,61 @@ def has_delete_permission(self, request, obj): # return format_pre_json(instance.feedback) -class ApplyJobAdmin(admin.ModelAdmin): +class JobAdmin(admin.ModelAdmin): list_display = ( "id", "project__owner", "project__name", + "type", "status", + "created_by__link", "created_at", "updated_at", ) - - list_filter = ("status", "updated_at") + list_filter = ("type", "status", "updated_at") + list_select_related = ("project", "project__owner", "created_by") + exclude = ("feedback", "output") ordering = ("-updated_at",) search_fields = ( "project__name__iexact", "project__owner__username__iexact", - "deltas_to_apply__id__startswith", "id", ) readonly_fields = ( + "project", + "status", + "type", "created_at", "updated_at", "started_at", "finished_at", + "output__pre", + "feedback__pre", ) - inlines = [ - DeltaInline, - ] + + def get_object(self, request, object_id, from_field=None): + obj = super().get_object(request, object_id, from_field) + if obj and obj.type == Job.Type.DELTA_APPLY: + obj = ApplyJob.objects.get(pk=obj.pk) + return obj + + def get_inline_instances(self, request, obj=None): + inline_instances = super().get_inline_instances(request, obj) + + if isinstance(obj, ApplyJob): + for inline_instance in inline_instances: + if inline_instance.parent_model == Job: + inline_instance.parent_model = ApplyJob + + return inline_instances + + def get_inlines(self, request, obj=None): + inlines = [*super().get_inlines(request, obj)] + + if obj and obj.type == Job.Type.DELTA_APPLY: + inlines.append(DeltaInline) + + return inlines def project__owner(self, instance): return model_admin_url(instance.project.owner) @@ -407,6 +465,11 @@ def project__name(self, instance): project__name.admin_order_field = "project__name" + def created_by__link(self, instance): + return model_admin_url(instance.created_by) + + created_by__link.admin_order_field = "created_by" + def has_add_permission(self, request, obj=None): return False @@ -416,6 +479,12 @@ def has_change_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None): return False + def output__pre(self, instance): + return format_pre(instance.output) + + def feedback__pre(self, instance): + return format_pre_json(instance.feedback) + class ApplyJobDeltaInline(admin.TabularInline): model = ApplyJobDelta @@ -554,122 +623,6 @@ def response_change(self, request, delta): return super().response_change(request, delta) -class PackageJobAdmin(admin.ModelAdmin): - list_display = ( - "id", - "project__owner", - "project__name", - "status", - "created_at", - "updated_at", - ) - list_filter = ("status", "updated_at") - list_select_related = ("project", "project__owner") - actions = None - exclude = ("feedback", "output") - - readonly_fields = ( - "project", - "status", - "created_at", - "updated_at", - "started_at", - "finished_at", - "output__pre", - "feedback__pre", - ) - - search_fields = ( - "id", - "feedback__icontains", - "project__name__iexact", - "project__owner__username__iexact", - ) - - ordering = ("-updated_at",) - - def project__owner(self, instance): - return model_admin_url(instance.project.owner) - - project__owner.admin_order_field = "project__owner" - - def project__name(self, instance): - return model_admin_url(instance.project, instance.project.name) - - project__name.admin_order_field = "project__name" - - def output__pre(self, instance): - return format_pre(instance.output) - - def feedback__pre(self, instance): - return format_pre_json(instance.feedback) - - # This will disable add functionality - def has_add_permission(self, request): - return False - - def has_change_permission(self, request, obj=None): - return False - - -class ProcessProjectfileJobAdmin(admin.ModelAdmin): - list_display = ( - "id", - "project__owner", - "project__name", - "status", - "created_at", - "updated_at", - ) - list_filter = ("status", "updated_at") - list_select_related = ("project", "project__owner") - actions = None - exclude = ("feedback", "output") - - readonly_fields = ( - "project", - "status", - "created_at", - "updated_at", - "started_at", - "finished_at", - "output__pre", - "feedback__pre", - ) - - search_fields = ( - "id", - "feedback__icontains", - "project__name__iexact", - "project__owner__username__iexact", - ) - - ordering = ("-updated_at",) - - def project__owner(self, instance): - return model_admin_url(instance.project.owner) - - project__owner.admin_order_field = "project__owner" - - def project__name(self, instance): - return model_admin_url(instance.project, instance.project.name) - - project__name.admin_order_field = "project__name" - - def output__pre(self, instance): - return format_pre(instance.output) - - def feedback__pre(self, instance): - return format_pre_json(instance.feedback) - - # This will disable add functionality - def has_add_permission(self, request): - return False - - def has_change_permission(self, request, obj=None): - return False - - class GeodbAdmin(admin.ModelAdmin): list_filter = ("created_at", "hostname") list_display = ( @@ -754,7 +707,6 @@ class OrganizationAdmin(admin.ModelAdmin): "email", "organization_owner__link", "date_joined", - "useraccount", ) search_fields = ( @@ -762,6 +714,8 @@ class OrganizationAdmin(admin.ModelAdmin): "organization_owner__username__icontains", ) + list_select_related = ("organization_owner",) + list_filter = ("date_joined",) def organization_owner__link(self, instance): @@ -823,17 +777,22 @@ def save_model(self, request, obj, form, change): obj.save() +class InvitationAdmin(InvitationAdminBase): + list_display = ("email", "inviter", "created", "sent", "accepted") + list_select_related = ("inviter",) + list_filter = ( + "accepted", + "created", + "sent", + ) + search_fields = ("email__icontains", "inviter__username__iexact") + + +admin.site.register(Invitation, InvitationAdmin) admin.site.register(User, UserAdmin) admin.site.register(Organization, OrganizationAdmin) admin.site.register(Team, TeamAdmin) admin.site.register(Project, ProjectAdmin) admin.site.register(Delta, DeltaAdmin) -admin.site.register(ApplyJob, ApplyJobAdmin) -admin.site.register(PackageJob, PackageJobAdmin) -admin.site.register(ProcessProjectfileJob, ProcessProjectfileJobAdmin) +admin.site.register(Job, JobAdmin) admin.site.register(Geodb, GeodbAdmin) - -admin.site.unregister(Group) -admin.site.unregister(SocialAccount) -admin.site.unregister(SocialApp) -admin.site.unregister(SocialToken) diff --git a/docker-app/qfieldcloud/core/tests/test_admin.py b/docker-app/qfieldcloud/core/tests/test_admin.py new file mode 100644 index 000000000..a7121c0fa --- /dev/null +++ b/docker-app/qfieldcloud/core/tests/test_admin.py @@ -0,0 +1,114 @@ +import logging + +from bs4 import BeautifulSoup +from django.conf import settings +from django.test.testcases import TransactionTestCase +from django.urls.resolvers import URLPattern, URLResolver + +from ..models import Delta, Organization, ProcessProjectfileJob, Project, Team, User +from .utils import setup_subscription_plans + +logging.disable(logging.CRITICAL) + + +def list_urls(url_items, prefixes=None): + if prefixes is None: + prefixes = [] + + if not url_items: + return + + url_item = url_items[0] + + if isinstance(url_item, URLPattern): + yield prefixes + [str(url_item.pattern)] + elif isinstance(url_item, URLResolver): + yield from list_urls(url_item.url_patterns, prefixes + [str(url_item.pattern)]) + yield from list_urls(url_items[1:], prefixes) + + +class QfcTestCase(TransactionTestCase): + def setUp(self): + setup_subscription_plans() + self.superuser = User.objects.create_superuser( + username="superuser", password="secret", email="super@example.com" + ) + + # we need to create these objects to be able to test sort by column + self.project = Project.objects.create( + name="project", + owner=self.superuser, + ) + self.organization = Organization.objects.create( + username="organization", + organization_owner=self.superuser, + ) + self.team = Team.objects.create( + username="@organization/team", + team_organization=self.organization, + ) + self.job = ProcessProjectfileJob.objects.create( + project=self.project, + created_by=self.superuser, + ) + self.delta = Delta.objects.create( + deltafile_id="f85b4d28-2444-40ce-95a8-6502bf4f00d9", + project=self.project, + created_by=self.superuser, + content={}, + ) + + def test_admin_opens(self): + skip_urls = ( + "/admin/login/", + "/admin/logout/", + "/admin/password_change/", + "/admin/password_change/done/", + "/admin/autocomplete/", + "/admin/core/delta/add/", + "/admin/core/job/add/", + "/admin/axes/accessattempt/add/", + "/admin/axes/accesslog/add/", + ) + # TODO make tests pass for these srotable URLs + skip_sort_urls = ("/admin/django_cron/cronjoblog/?o=4",) + + self.client.force_login(self.superuser) + + urlconf = __import__(settings.ROOT_URLCONF, {}, {}, [""]) + + for url_item in list_urls(urlconf.urlpatterns): + if url_item[0] != "admin/": + continue + + url = "/" + "".join(url_item) + + # skip if the URL pattern contains placeholders, e.g. /admin/app/model//edit + if "<" in url: + continue + + if url in skip_urls: + continue + + # get page without any sorting + resp = self.client.get(f"{url}?o=") + self.assertEqual( + resp.status_code, + 200, + f'Failed to open "{url}", got HTTP {resp.status_code}.', + ) + + # check all different sort columns + soup = BeautifulSoup(resp.content, "html.parser") + for anchor in soup.select("th.sortable a"): + sort_url = f"{url}{anchor.get('href')}" + + if sort_url in skip_sort_urls: + continue + + resp = self.client.get(sort_url) + self.assertEqual( + resp.status_code, + 200, + f'Failed to sort "{sort_url}", got HTTP {resp.status_code}.', + ) diff --git a/docker-app/qfieldcloud/notifs/admin.py b/docker-app/qfieldcloud/notifs/admin.py index 82a9451f4..022198a1b 100644 --- a/docker-app/qfieldcloud/notifs/admin.py +++ b/docker-app/qfieldcloud/notifs/admin.py @@ -1,12 +1,26 @@ -from notifications.admin import NotificationAdmin - -# Override django-notification's list display -NotificationAdmin.list_display = ( - "actor", - "verb", - "action_object", - "target", - "timestamp", - "recipient", - "unread", -) +from django.contrib import admin +from notifications.admin import NotificationAdmin as NotificationAdminBase +from notifications.models import Notification + +admin.site.unregister(Notification) + + +class NotificationAdmin(NotificationAdminBase): + list_display = ( + "actor", + "verb", + "action_object", + "target", + "timestamp", + "recipient", + "unread", + ) + + list_select_related = ("recipient",) + + list_per_page = 10 + + search_fields = ("recipient__username__icontains",) + + +admin.site.register(Notification, NotificationAdmin) From 7087d02d87b69e97eb62cd45b040764c3a3946b5 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Thu, 8 Dec 2022 13:47:12 +0200 Subject: [PATCH 13/16] Merge pull request #452 from opengisch/fix_dangling_qgis --- docker-app/requirements.txt | 1 + docker-app/worker_wrapper/wrapper.py | 35 ++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docker-app/requirements.txt b/docker-app/requirements.txt index 2eae93739..003995225 100644 --- a/docker-app/requirements.txt +++ b/docker-app/requirements.txt @@ -83,6 +83,7 @@ six==1.16.0 soupsieve==2.3.1 sqlparse==0.4.2 swapper==1.3.0 +tenacity==8.1.0 typing-extensions==4.0.1 uritemplate==4.1.1 urllib3==1.26.7 diff --git a/docker-app/worker_wrapper/wrapper.py b/docker-app/worker_wrapper/wrapper.py index a2fa36a20..50fc092bd 100644 --- a/docker-app/worker_wrapper/wrapper.py +++ b/docker-app/worker_wrapper/wrapper.py @@ -29,9 +29,16 @@ ) from qfieldcloud.core.utils import get_qgis_project_file from qfieldcloud.core.utils2 import storage +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_random_exponential, +) logger = logging.getLogger(__name__) +RETRY_COUNT = 5 TIMEOUT_ERROR_EXIT_CODE = -1 QGIS_CONTAINER_NAME = os.environ.get("QGIS_CONTAINER_NAME", None) QFIELDCLOUD_HOST = os.environ.get("QFIELDCLOUD_HOST", None) @@ -256,14 +263,34 @@ def _run_docker( response = {"StatusCode": TIMEOUT_ERROR_EXIT_CODE} try: - # will throw an ConnectionError, but the container is still alive + # will throw an `requests.exceptions.ConnectionError`, but the container is still alive response = container.wait(timeout=self.container_timeout_secs) except Exception as err: logger.exception("Timeout error.", exc_info=err) - logs = container.logs() - container.stop() - container.remove() + logs = b"" + # Retry reading the logs, as it may fail + # NOTE when reading the logs of a finished container, it might timeout with an ``. + # This leads to exception and prevents the container to be removed few lines below. + # Therefore try reading the logs, as they are important, and if it fails, just use a + # generic "failed to read logs" message. + # Similar issue here: https://github.com/docker/docker-py/issues/2266 + + retriable = retry( + wait=wait_random_exponential(max=10), + stop=stop_after_attempt(RETRY_COUNT), + retry=retry_if_exception_type(requests.exceptions.ConnectionError), + reraise=True, + ) + + try: + logs = retriable(lambda: container.logs())() + except requests.exceptions.ConnectionError: + logs = b"[QFC/Worker/1001] Failed to read logs." + + retriable(lambda: container.stop())() + retriable(lambda: container.remove())() + logger.info( f"Finished execution with code {response['StatusCode']}, logs:\n{logs}" ) From 089770a824fbafa16db0eed64c0a016828bdf25e Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Thu, 8 Dec 2022 13:47:41 +0200 Subject: [PATCH 14/16] Merge pull request #455 from opengisch/qgis_limits --- docker-app/qfieldcloud/settings.py | 16 ++++++++++++++-- docker-app/worker_wrapper/wrapper.py | 2 ++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docker-app/qfieldcloud/settings.py b/docker-app/qfieldcloud/settings.py index 6193bff17..c29c795d7 100644 --- a/docker-app/qfieldcloud/settings.py +++ b/docker-app/qfieldcloud/settings.py @@ -347,9 +347,17 @@ CONSTANCE_BACKEND = "qfieldcloud.core.constance_backends.DatabaseBackend" CONSTANCE_CONFIG = { "WORKER_TIMEOUT_S": ( - 60, + 600, "Timeout of the workers before being terminated by the wrapper in seconds.", ), + "WORKER_QGIS_MEMORY_LIMIT": ( + "1000m", + "Maximum memory for each QGIS worker container.", + ), + "WORKER_QGIS_CPU_SHARES": ( + 512, + "Share of CPUs for each QGIS worker container. By default all containers have value 1024 set by docker.", + ), } CONSTANCE_ADDITIONAL_FIELDS = { "textarea": [ @@ -360,5 +368,9 @@ ] } CONSTANCE_CONFIG_FIELDSETS = { - "Worker": ("WORKER_TIMEOUT_S",), + "Worker": ( + "WORKER_TIMEOUT_S", + "WORKER_QGIS_MEMORY_LIMIT", + "WORKER_QGIS_CPU_SHARES", + ), } diff --git a/docker-app/worker_wrapper/wrapper.py b/docker-app/worker_wrapper/wrapper.py index 50fc092bd..bcb200f93 100644 --- a/docker-app/worker_wrapper/wrapper.py +++ b/docker-app/worker_wrapper/wrapper.py @@ -256,6 +256,8 @@ def _run_docker( # auto_remove=True, network=os.environ.get("QFIELDCLOUD_DEFAULT_NETWORK"), detach=True, + mem_limit=config.WORKER_QGIS_MEMORY_LIMIT, + cpu_shares=config.WORKER_QGIS_CPU_SHARES, ) logger.info(f"Starting worker {container.id} ...") From fc551ef5c26c5c9a6460d24eb65cd4fe0e2203b0 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Sat, 10 Dec 2022 02:27:16 +0200 Subject: [PATCH 15/16] Merge pull request #457 from opengisch/no_collectstatic_js --- docker-compose.override.dev.yml | 5 +++++ docker-compose.override.prod.yml | 7 +++++++ docker-compose.yml | 2 -- 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 docker-compose.override.prod.yml diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index 0da9a32d6..7c6cf1632 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -21,5 +21,10 @@ services: ports: - ${GEODB_PORT}:5432 + nginx: + volumes: + - static_volume:/var/www/html/staticfiles/ + - media_volume:/var/www/html/mediafiles/ + volumes: geodb_data: diff --git a/docker-compose.override.prod.yml b/docker-compose.override.prod.yml new file mode 100644 index 000000000..c2f4efdd6 --- /dev/null +++ b/docker-compose.override.prod.yml @@ -0,0 +1,7 @@ +version: '3.9' + +services: + nginx: + volumes: + - static_volume:/var/www/html/staticfiles/ + - media_volume:/var/www/html/mediafiles/ diff --git a/docker-compose.yml b/docker-compose.yml index 5056d5550..86f6f2da7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -86,8 +86,6 @@ services: image: nginx:stable restart: unless-stopped volumes: - - static_volume:/var/www/html/staticfiles/ - - media_volume:/var/www/html/mediafiles/ - ./conf/nginx/pages/:/var/www/html/pages/ - ./conf/nginx/templates/:/etc/nginx/templates/ - ./conf/nginx/certs/:/etc/nginx/certs/:ro From bb1d5cbcc25cc2beb5a16ca06bbe5bbc11395229 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Sat, 17 Dec 2022 15:22:21 +0200 Subject: [PATCH 16/16] Merge pull request #460 from opengisch/fix_build_network --- README.md | 6 ++++ conf/nginx/templates/default.conf.template | 34 ++++++++++++++++++---- docker-compose.yml | 6 ++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2e6be76b1..2c27049ea 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,12 @@ Docker logs are managed by docker in the default way. To read the logs: docker-compose logs +For great `nginx` logs, use: + + QFC_JQ='[.ts, .ip, (.method + " " + (.status|tostring) + " " + (.resp_time|tostring) + "s"), .uri, "I " + (.request_length|tostring) + " O " + (.resp_body_size|tostring), "C " + (.upstream_connect_time|tostring) + "s", "H " + (.upstream_header_time|tostring) + "s", "R " + (.upstream_response_time|tostring) + "s", .user_agent] | @tsv' + docker compose logs nginx -f --no-log-prefix | grep ':"nginx"' | jq -r $QFC_JQ + + ### Geodb The geodb (database for the users projects data) is installed on diff --git a/conf/nginx/templates/default.conf.template b/conf/nginx/templates/default.conf.template index 859c63ee9..2c4a41211 100644 --- a/conf/nginx/templates/default.conf.template +++ b/conf/nginx/templates/default.conf.template @@ -1,7 +1,30 @@ -log_format upstreamlog '[$time_local] $remote_addr - $remote_user ' - 'to: $upstream_addr "$request" $status $body_bytes_sent ' - '"$http_referer" "$http_user_agent"' - 'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'; +map "$time_iso8601 # $msec" $time_iso8601_ms { + "~([^+]+)\+([\d:]+?) # \d+?\.(\d+)" "$1.$3+$2"; +} + +log_format json-logger escape=json +'{' + '"ts":"$time_iso8601_ms",' + '"ip":"$remote_addr",' + '"method":"$request_method",' + '"status":$status,' + '"resp_time":$request_time,' + '"request_length":$request_length,' + '"resp_body_size":$body_bytes_sent,' + '"uri":"$request_uri",' + '"connection": "$connection",' + '"connection_requests": "$connection_requests",' + '"user_agent":"$http_user_agent",' + '"host":"$http_host",' + '"user":"$remote_user",' + '"upstream_addr":"$upstream_addr",' + '"upstream_connect_time":"$upstream_connect_time",' + '"upstream_header_time":"$upstream_header_time",' + '"upstream_response_time":"$upstream_response_time",' + '"source":"nginx"' +'}'; + + upstream django { server app:8000 fail_timeout=0; @@ -33,6 +56,8 @@ server { ssl_certificate certs/${QFIELDCLOUD_HOST}.pem; ssl_certificate_key certs/${QFIELDCLOUD_HOST}-key.pem; + access_log /var/log/nginx/access.log json-logger; + server_name ${QFIELDCLOUD_HOST}; client_max_body_size 10G; keepalive_timeout 5; @@ -85,7 +110,6 @@ server { # Only allow internal redirects internal; - access_log /var/log/nginx/access.log upstreamlog; set $redirect_uri "$upstream_http_redirect_uri"; # required DNS diff --git a/docker-compose.yml b/docker-compose.yml index 86f6f2da7..e685d5d3a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ services: app: &default-django build: context: ./docker-app + network: host restart: unless-stopped command: > gunicorn @@ -123,6 +124,7 @@ services: qgis: build: context: ./docker-qgis + network: host tty: true command: bash -c "echo QGIS built" logging: *default-logging @@ -130,6 +132,7 @@ services: redis: build: context: ./docker-redis + network: host args: REDIS_PASSWORD: ${REDIS_PASSWORD} restart: unless-stopped @@ -139,6 +142,9 @@ services: worker_wrapper: <<: *default-django + build: + context: ./docker-app + network: host command: python manage.py dequeue user: root # TODO change me to least privileged docker-capable user on the host (/!\ docker users!=hosts users, use UID rather than username) volumes: