Merge commit 'f7ccf4d6c3f289fea5ec90e87f56731741aea6bf' into release

.env.example

@@ -52,8 +52,14 @@ GEODB_USER=postgres
 GEODB_PASSWORD=KUAa7h!G&wQEmkS3
 GEODB_DB=postgres
 
+# Sentry DSN. Missing value disables Sentry logging. Can be found on https://opengisch.sentry.io/settings/projects/qfieldcloud/keys/ .
+# DEFAULT: <NO VALUE>
 SENTRY_DSN=
 
+# Sentry sample rate between 0 and 1. Read more on https://docs.sentry.io/platforms/python/configuration/sampling/ .
+# DEFAULT: 1
+SENTRY_SAMPLE_RATE=1
+
 REDIS_PASSWORD=change_me_with_a_very_loooooooooooong_password
 REDIS_PORT=6379
 

conf/nginx/templates/default.conf.template

@@ -21,6 +21,7 @@ log_format json-logger escape=json
   '"upstream_connect_time":"$upstream_connect_time",'
   '"upstream_header_time":"$upstream_header_time",'
   '"upstream_response_time":"$upstream_response_time",'
+  '"request_id":"$request_id",'
   '"source":"nginx"'
 '}';
 

docker-app/Dockerfile

@@ -74,6 +74,7 @@ ENTRYPOINT ["/entrypoint.sh"]
 
 # a separate stage for webserver runtime environment
 FROM base as webserver_runtime
+ENV LOGGER_SOURCE=app
 COPY ./requirements_runtime.txt .
 RUN pip3 install -r requirements_runtime.txt
 EXPOSE 8000
@@ -88,6 +89,7 @@ USER app
 
 # a separate stage for webserver test environment
 FROM base as webserver_test
+ENV LOGGER_SOURCE=app
 COPY ./requirements_test.txt .
 RUN pip3 install -r requirements_test.txt
 EXPOSE 8000
@@ -102,6 +104,7 @@ USER app
 
 # a separate stage for worker wrapper runtime environment
 FROM base as worker_wrapper_runtime
+ENV LOGGER_SOURCE=worker_wrapper
 COPY ./requirements_worker_wrapper.txt .
 RUN pip3 install -r requirements_worker_wrapper.txt
 

docker-app/qfieldcloud/core/admin.py

@@ -11,7 +11,9 @@
 from django.contrib.auth.models import Group
 from django.core.exceptions import PermissionDenied
 from django.db.models.fields.json import JSONField
+from django.db.models.functions import Lower
 from django.forms import ModelForm, fields, widgets
+from django.http import HttpRequest
 from django.http.response import Http404, HttpResponseRedirect
 from django.shortcuts import resolve_url
 from django.template.response import TemplateResponse
@@ -245,7 +247,7 @@ class PersonAdmin(admin.ModelAdmin):
         "is_staff",
     )
 
-    search_fields = ("username__icontains",)
+    search_fields = ("username__icontains", "email__iexact")
 
     fields = (
         "username",
@@ -330,8 +332,18 @@ def password_reset_url(self, request, user_id, form_url=""):
 
 class ProjectCollaboratorInline(admin.TabularInline):
     model = ProjectCollaborator
+
     extra = 0
 
+    readonly_fields = (
+        "created_by",
+        "updated_by",
+        "created_at",
+        "updated_at",
+    )
+
+    autocomplete_fields = ("collaborator",)
+
 
 class ProjectFilesWidget(widgets.Input):
     template_name = "admin/project_files_widget.html"
@@ -371,7 +383,7 @@ class ProjectAdmin(admin.ModelAdmin):
         "is_public",
         "owner",
         "project_filename",
-        "storage_size_mb",
+        "file_storage_bytes",
         "created_at",
         "updated_at",
         "data_last_updated_at",
@@ -381,7 +393,7 @@ class ProjectAdmin(admin.ModelAdmin):
     )
     readonly_fields = (
         "id",
-        "storage_size_mb",
+        "file_storage_bytes",
         "created_at",
         "updated_at",
         "data_last_updated_at",
@@ -394,6 +406,7 @@ class ProjectAdmin(admin.ModelAdmin):
         "name__icontains",
         "owner__username__iexact",
     )
+    autocomplete_fields = ("owner",)
 
     ordering = ("-updated_at",)
 
@@ -433,6 +446,17 @@ def project_details__pre(self, instance):
 
         return format_pre_json(instance.project_details)
 
+    def save_formset(self, request, form, formset, change):
+        for form_obj in formset:
+            if isinstance(form_obj.instance, ProjectCollaborator):
+                # add created_by only if it's a newly created collaborator
+                if form_obj.instance.id is None:
+                    form_obj.instance.created_by = request.user
+
+                form_obj.instance.updated_by = request.user
+
+        super().save_formset(request, form, formset, change)
+
 
 class DeltaInline(admin.TabularInline):
     model = ApplyJob.deltas_to_apply.through
@@ -726,6 +750,8 @@ class OrganizationMemberInline(admin.TabularInline):
     fk_name = "organization"
     extra = 0
 
+    autocomplete_fields = ("member",)
+
 
 class TeamInline(admin.TabularInline):
     model = Team
@@ -767,12 +793,16 @@ class OrganizationAdmin(admin.ModelAdmin):
     search_fields = (
         "username__icontains",
         "organization_owner__username__icontains",
+        "email__iexact",
+        "organization_owner__email__iexact",
     )
 
     list_select_related = ("organization_owner",)
 
     list_filter = ("date_joined",)
 
+    autocomplete_fields = ("organization_owner",)
+
     def organization_owner__link(self, instance):
         return model_admin_url(
             instance.organization_owner, instance.organization_owner.username
@@ -808,6 +838,8 @@ class TeamMemberInline(admin.TabularInline):
     fk_name = "team"
     extra = 0
 
+    autocomplete_fields = ("member",)
+
 
 class TeamAdmin(admin.ModelAdmin):
     inlines = (TeamMemberInline,)
@@ -826,6 +858,8 @@ class TeamAdmin(admin.ModelAdmin):
 
     list_filter = ("date_joined",)
 
+    autocomplete_fields = ("team_organization",)
+
     def save_model(self, request, obj, form, change):
         if not obj.username.startswith("@"):
             obj.username = f"@{obj.team_organization.username}/{obj.username}"
@@ -843,6 +877,36 @@ class InvitationAdmin(InvitationAdminBase):
     search_fields = ("email__icontains", "inviter__username__iexact")
 
 
+class UserAccountAdmin(admin.ModelAdmin):
+    """The sole purpose of this admin module is only to support autocomplete fields in Django admin."""
+
+    ordering = (Lower("user__username"),)
+    search_fields = ("user__username__icontains",)
+    list_select_related = ("user",)
+
+    def has_module_permission(self, request: HttpRequest) -> bool:
+        # hide this module from Django admin, it is accessible via "Person" and "Organization" as inline edit
+        return False
+
+
+class UserAdmin(admin.ModelAdmin):
+    """The sole purpose of this admin module is only to support autocomplete fields in Django admin."""
+
+    ordering = (Lower("username"),)
+    search_fields = ("username__icontains",)
+
+    def get_queryset(self, request: HttpRequest):
+        return (
+            super()
+            .get_queryset(request)
+            .filter(type__in=(User.Type.PERSON, User.Type.ORGANIZATION))
+        )
+
+    def has_module_permission(self, request: HttpRequest) -> bool:
+        # hide this module from Django admin, it is accessible via "Person" and "Organization" as inline edit
+        return False
+
+
 admin.site.register(Invitation, InvitationAdmin)
 admin.site.register(Person, PersonAdmin)
 admin.site.register(Organization, OrganizationAdmin)
@@ -851,3 +915,7 @@ class InvitationAdmin(InvitationAdminBase):
 admin.site.register(Delta, DeltaAdmin)
 admin.site.register(Job, JobAdmin)
 admin.site.register(Geodb, GeodbAdmin)
+
+# The sole purpose of the `User` and `UserAccount` admin modules is only to support autocomplete fields in Django admin
+admin.site.register(User, UserAdmin)
+admin.site.register(UserAccount, UserAccountAdmin)

docker-app/qfieldcloud/core/logging/formatters.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 
 import json_log_formatter
+from django.conf import settings
 from django.core.handlers.wsgi import WSGIRequest
 from django.core.serializers.json import DjangoJSONEncoder
 
@@ -23,7 +24,7 @@ def json_record(self, message, extra, record):
         :return: Dictionary which will be passed to JSON lib.
 
         """
-        if "ts" in extra:
+        if "ts" not in extra:
             extra["ts"] = datetime.utcnow()
 
         # Include builtins
@@ -34,6 +35,7 @@ def json_record(self, message, extra, record):
         extra["filename"] = record.filename
         extra["lineno"] = record.lineno
         extra["thread"] = record.thread
+        extra["source"] = settings.LOGGER_SOURCE
 
         if record.exc_info:
             extra["exc_info"] = self.formatException(record.exc_info)
@@ -49,11 +51,11 @@ def to_json(self, record):
         """
         try:
             return self.json_lib.dumps(
-                record, default=json_default, sort_keys=True, cls=JsonEncoder
+                record, default=json_default, cls=JsonEncoder, separators=(",", ":")
             )
         # ujson doesn't support default argument and raises TypeError.
         except TypeError:
-            return self.json_lib.dumps(record, cls=JsonEncoder)
+            return self.json_lib.dumps(record, cls=JsonEncoder, separators=(",", ":"))
 
 
 def json_default(obj):

docker-app/qfieldcloud/core/management/commands/calcprojectstorage.py

@@ -22,7 +22,7 @@ def handle(self, *args, **options):
             extra_filters["id"] = project_id
 
         if not project_id and not force_recalculate:
-            extra_filters["storage_size_mb"] = 0
+            extra_filters["file_storage_bytes"] = 0
 
         projects_qs = Project.objects.filter(
             project_filename__isnull=False,
@@ -36,5 +36,5 @@ def handle(self, *args, **options):
             )
             project.save(recompute_storage=True)
             print(
-                f'Project files storage size for "{project.id}" is {project.storage_size_mb}MB'
+                f'Project files storage size for "{project.id}" is {project.file_storage_bytes} bytes.'
             )

docker-app/qfieldcloud/core/migrations/0060_alter_project_storage_size_mb.py

@@ -0,0 +1,35 @@
+# Generated by Django 3.2.17 on 2023-02-10 00:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0059_auto_20221028_1806"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="project",
+            old_name="storage_size_mb",
+            new_name="file_storage_bytes",
+        ),
+        migrations.RunSQL(
+            """
+                UPDATE core_project
+                SET
+                    file_storage_bytes = file_storage_bytes * 1000 * 1000
+            """,
+            """
+                UPDATE core_project
+                SET
+                    file_storage_bytes = file_storage_bytes / 1000 / 1000
+            """,
+        ),
+        migrations.AlterField(
+            model_name="project",
+            name="file_storage_bytes",
+            field=models.PositiveBigIntegerField(default=0),
+        ),
+    ]