diff --git a/.env.example b/.env.example index b45f1cdfe..0be1758ee 100644 --- a/.env.example +++ b/.env.example @@ -52,8 +52,14 @@ GEODB_USER=postgres GEODB_PASSWORD=KUAa7h!G&wQEmkS3 GEODB_DB=postgres +# Sentry DSN. Missing value disables Sentry logging. Can be found on https://opengisch.sentry.io/settings/projects/qfieldcloud/keys/ . +# DEFAULT: SENTRY_DSN= +# Sentry sample rate between 0 and 1. Read more on https://docs.sentry.io/platforms/python/configuration/sampling/ . +# DEFAULT: 1 +SENTRY_SAMPLE_RATE=1 + REDIS_PASSWORD=change_me_with_a_very_loooooooooooong_password REDIS_PORT=6379 diff --git a/conf/nginx/templates/default.conf.template b/conf/nginx/templates/default.conf.template index 33ad19ded..a29624732 100644 --- a/conf/nginx/templates/default.conf.template +++ b/conf/nginx/templates/default.conf.template @@ -21,6 +21,7 @@ log_format json-logger escape=json '"upstream_connect_time":"$upstream_connect_time",' '"upstream_header_time":"$upstream_header_time",' '"upstream_response_time":"$upstream_response_time",' + '"request_id":"$request_id",' '"source":"nginx"' '}'; diff --git a/docker-app/Dockerfile b/docker-app/Dockerfile index ff99bab61..874c165a6 100644 --- a/docker-app/Dockerfile +++ b/docker-app/Dockerfile @@ -74,6 +74,7 @@ ENTRYPOINT ["/entrypoint.sh"] # a separate stage for webserver runtime environment FROM base as webserver_runtime +ENV LOGGER_SOURCE=app COPY ./requirements_runtime.txt . RUN pip3 install -r requirements_runtime.txt EXPOSE 8000 @@ -88,6 +89,7 @@ USER app # a separate stage for webserver test environment FROM base as webserver_test +ENV LOGGER_SOURCE=app COPY ./requirements_test.txt . RUN pip3 install -r requirements_test.txt EXPOSE 8000 @@ -102,6 +104,7 @@ USER app # a separate stage for worker wrapper runtime environment FROM base as worker_wrapper_runtime +ENV LOGGER_SOURCE=worker_wrapper COPY ./requirements_worker_wrapper.txt . RUN pip3 install -r requirements_worker_wrapper.txt diff --git a/docker-app/qfieldcloud/core/admin.py b/docker-app/qfieldcloud/core/admin.py index b580a6f7b..18127f5b8 100644 --- a/docker-app/qfieldcloud/core/admin.py +++ b/docker-app/qfieldcloud/core/admin.py @@ -11,7 +11,9 @@ from django.contrib.auth.models import Group from django.core.exceptions import PermissionDenied from django.db.models.fields.json import JSONField +from django.db.models.functions import Lower from django.forms import ModelForm, fields, widgets +from django.http import HttpRequest from django.http.response import Http404, HttpResponseRedirect from django.shortcuts import resolve_url from django.template.response import TemplateResponse @@ -245,7 +247,7 @@ class PersonAdmin(admin.ModelAdmin): "is_staff", ) - search_fields = ("username__icontains",) + search_fields = ("username__icontains", "email__iexact") fields = ( "username", @@ -330,8 +332,18 @@ def password_reset_url(self, request, user_id, form_url=""): class ProjectCollaboratorInline(admin.TabularInline): model = ProjectCollaborator + extra = 0 + readonly_fields = ( + "created_by", + "updated_by", + "created_at", + "updated_at", + ) + + autocomplete_fields = ("collaborator",) + class ProjectFilesWidget(widgets.Input): template_name = "admin/project_files_widget.html" @@ -371,7 +383,7 @@ class ProjectAdmin(admin.ModelAdmin): "is_public", "owner", "project_filename", - "storage_size_mb", + "file_storage_bytes", "created_at", "updated_at", "data_last_updated_at", @@ -381,7 +393,7 @@ class ProjectAdmin(admin.ModelAdmin): ) readonly_fields = ( "id", - "storage_size_mb", + "file_storage_bytes", "created_at", "updated_at", "data_last_updated_at", @@ -394,6 +406,7 @@ class ProjectAdmin(admin.ModelAdmin): "name__icontains", "owner__username__iexact", ) + autocomplete_fields = ("owner",) ordering = ("-updated_at",) @@ -433,6 +446,17 @@ def project_details__pre(self, instance): return format_pre_json(instance.project_details) + def save_formset(self, request, form, formset, change): + for form_obj in formset: + if isinstance(form_obj.instance, ProjectCollaborator): + # add created_by only if it's a newly created collaborator + if form_obj.instance.id is None: + form_obj.instance.created_by = request.user + + form_obj.instance.updated_by = request.user + + super().save_formset(request, form, formset, change) + class DeltaInline(admin.TabularInline): model = ApplyJob.deltas_to_apply.through @@ -726,6 +750,8 @@ class OrganizationMemberInline(admin.TabularInline): fk_name = "organization" extra = 0 + autocomplete_fields = ("member",) + class TeamInline(admin.TabularInline): model = Team @@ -767,12 +793,16 @@ class OrganizationAdmin(admin.ModelAdmin): search_fields = ( "username__icontains", "organization_owner__username__icontains", + "email__iexact", + "organization_owner__email__iexact", ) list_select_related = ("organization_owner",) list_filter = ("date_joined",) + autocomplete_fields = ("organization_owner",) + def organization_owner__link(self, instance): return model_admin_url( instance.organization_owner, instance.organization_owner.username @@ -808,6 +838,8 @@ class TeamMemberInline(admin.TabularInline): fk_name = "team" extra = 0 + autocomplete_fields = ("member",) + class TeamAdmin(admin.ModelAdmin): inlines = (TeamMemberInline,) @@ -826,6 +858,8 @@ class TeamAdmin(admin.ModelAdmin): list_filter = ("date_joined",) + autocomplete_fields = ("team_organization",) + def save_model(self, request, obj, form, change): if not obj.username.startswith("@"): obj.username = f"@{obj.team_organization.username}/{obj.username}" @@ -843,6 +877,36 @@ class InvitationAdmin(InvitationAdminBase): search_fields = ("email__icontains", "inviter__username__iexact") +class UserAccountAdmin(admin.ModelAdmin): + """The sole purpose of this admin module is only to support autocomplete fields in Django admin.""" + + ordering = (Lower("user__username"),) + search_fields = ("user__username__icontains",) + list_select_related = ("user",) + + def has_module_permission(self, request: HttpRequest) -> bool: + # hide this module from Django admin, it is accessible via "Person" and "Organization" as inline edit + return False + + +class UserAdmin(admin.ModelAdmin): + """The sole purpose of this admin module is only to support autocomplete fields in Django admin.""" + + ordering = (Lower("username"),) + search_fields = ("username__icontains",) + + def get_queryset(self, request: HttpRequest): + return ( + super() + .get_queryset(request) + .filter(type__in=(User.Type.PERSON, User.Type.ORGANIZATION)) + ) + + def has_module_permission(self, request: HttpRequest) -> bool: + # hide this module from Django admin, it is accessible via "Person" and "Organization" as inline edit + return False + + admin.site.register(Invitation, InvitationAdmin) admin.site.register(Person, PersonAdmin) admin.site.register(Organization, OrganizationAdmin) @@ -851,3 +915,7 @@ class InvitationAdmin(InvitationAdminBase): admin.site.register(Delta, DeltaAdmin) admin.site.register(Job, JobAdmin) admin.site.register(Geodb, GeodbAdmin) + +# The sole purpose of the `User` and `UserAccount` admin modules is only to support autocomplete fields in Django admin +admin.site.register(User, UserAdmin) +admin.site.register(UserAccount, UserAccountAdmin) diff --git a/docker-app/qfieldcloud/core/logging/formatters.py b/docker-app/qfieldcloud/core/logging/formatters.py index 8d73c071b..f395c7578 100644 --- a/docker-app/qfieldcloud/core/logging/formatters.py +++ b/docker-app/qfieldcloud/core/logging/formatters.py @@ -1,6 +1,7 @@ from datetime import datetime import json_log_formatter +from django.conf import settings from django.core.handlers.wsgi import WSGIRequest from django.core.serializers.json import DjangoJSONEncoder @@ -23,7 +24,7 @@ def json_record(self, message, extra, record): :return: Dictionary which will be passed to JSON lib. """ - if "ts" in extra: + if "ts" not in extra: extra["ts"] = datetime.utcnow() # Include builtins @@ -34,6 +35,7 @@ def json_record(self, message, extra, record): extra["filename"] = record.filename extra["lineno"] = record.lineno extra["thread"] = record.thread + extra["source"] = settings.LOGGER_SOURCE if record.exc_info: extra["exc_info"] = self.formatException(record.exc_info) @@ -49,11 +51,11 @@ def to_json(self, record): """ try: return self.json_lib.dumps( - record, default=json_default, sort_keys=True, cls=JsonEncoder + record, default=json_default, cls=JsonEncoder, separators=(",", ":") ) # ujson doesn't support default argument and raises TypeError. except TypeError: - return self.json_lib.dumps(record, cls=JsonEncoder) + return self.json_lib.dumps(record, cls=JsonEncoder, separators=(",", ":")) def json_default(obj): diff --git a/docker-app/qfieldcloud/core/management/commands/calcprojectstorage.py b/docker-app/qfieldcloud/core/management/commands/calcprojectstorage.py index 4ee6677e3..b5aa3ddd4 100644 --- a/docker-app/qfieldcloud/core/management/commands/calcprojectstorage.py +++ b/docker-app/qfieldcloud/core/management/commands/calcprojectstorage.py @@ -22,7 +22,7 @@ def handle(self, *args, **options): extra_filters["id"] = project_id if not project_id and not force_recalculate: - extra_filters["storage_size_mb"] = 0 + extra_filters["file_storage_bytes"] = 0 projects_qs = Project.objects.filter( project_filename__isnull=False, @@ -36,5 +36,5 @@ def handle(self, *args, **options): ) project.save(recompute_storage=True) print( - f'Project files storage size for "{project.id}" is {project.storage_size_mb}MB' + f'Project files storage size for "{project.id}" is {project.file_storage_bytes} bytes.' ) diff --git a/docker-app/qfieldcloud/core/migrations/0060_alter_project_storage_size_mb.py b/docker-app/qfieldcloud/core/migrations/0060_alter_project_storage_size_mb.py new file mode 100644 index 000000000..bfa2e79c9 --- /dev/null +++ b/docker-app/qfieldcloud/core/migrations/0060_alter_project_storage_size_mb.py @@ -0,0 +1,35 @@ +# Generated by Django 3.2.17 on 2023-02-10 00:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0059_auto_20221028_1806"), + ] + + operations = [ + migrations.RenameField( + model_name="project", + old_name="storage_size_mb", + new_name="file_storage_bytes", + ), + migrations.RunSQL( + """ + UPDATE core_project + SET + file_storage_bytes = file_storage_bytes * 1000 * 1000 + """, + """ + UPDATE core_project + SET + file_storage_bytes = file_storage_bytes / 1000 / 1000 + """, + ), + migrations.AlterField( + model_name="project", + name="file_storage_bytes", + field=models.PositiveBigIntegerField(default=0), + ), + ] diff --git a/docker-app/qfieldcloud/core/migrations/0061_add_incognito_and_audit_to_collaborators.py b/docker-app/qfieldcloud/core/migrations/0061_add_incognito_and_audit_to_collaborators.py new file mode 100644 index 000000000..6feecf816 --- /dev/null +++ b/docker-app/qfieldcloud/core/migrations/0061_add_incognito_and_audit_to_collaborators.py @@ -0,0 +1,69 @@ +# Generated by Django 3.2.17 on 2023-02-14 09:21 + +import django.db.models.deletion +import django.utils.timezone +import migrate_sql.operations +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0060_alter_project_storage_size_mb"), + ] + + operations = [ + migrations.AddField( + model_name="projectcollaborator", + name="created_at", + field=models.DateTimeField( + auto_now_add=True, default=django.utils.timezone.now + ), + preserve_default=False, + ), + migrations.AddField( + model_name="projectcollaborator", + name="created_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="core.person", + ), + ), + migrations.AddField( + model_name="projectcollaborator", + name="updated_at", + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name="projectcollaborator", + name="updated_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="core.person", + ), + ), + migrations.AddField( + model_name="projectcollaborator", + name="is_incognito", + field=models.BooleanField( + default=False, + help_text="If a collaborator is marked as incognito, they will work as normal, but will not be listed in the UI or accounted in the subscription as active users. Used to add OPENGIS.ch support members to projects.", + ), + ), + migrate_sql.operations.ReverseAlterSQL( + name="projects_with_roles_vw", + sql="\n DROP VIEW projects_with_roles_vw;\n ", + reverse_sql='\n CREATE OR REPLACE VIEW projects_with_roles_vw AS\n\n WITH project_owner AS (\n SELECT\n 1 AS rank,\n P1."id" AS "project_id",\n P1."owner_id" AS "user_id",\n \'admin\' AS "name",\n \'project_owner\' AS "origin"\n FROM\n "core_project" P1\n INNER JOIN "core_user" U1 ON (P1."owner_id" = U1."id")\n WHERE\n U1."type" = 1\n ),\n organization_owner AS (\n SELECT\n 2 AS rank,\n P1."id" AS "project_id",\n O1."organization_owner_id" AS "user_id",\n \'admin\' AS "name",\n \'organization_owner\' AS "origin"\n FROM\n "core_organization" O1\n INNER JOIN "core_project" P1 ON (P1."owner_id" = O1."user_ptr_id")\n ),\n organization_admin AS (\n SELECT\n 3 AS rank,\n P1."id" AS "project_id",\n OM1."member_id" AS "user_id",\n \'admin\' AS "name",\n \'organization_admin\' AS "origin"\n FROM\n "core_organizationmember" OM1\n INNER JOIN "core_project" P1 ON (P1."owner_id" = OM1."organization_id")\n WHERE\n (\n OM1."role" = \'admin\'\n )\n ),\n project_collaborator AS (\n SELECT\n 4 AS rank,\n C1."project_id",\n C1."collaborator_id" AS "user_id",\n C1."role" AS "name",\n \'collaborator\' AS "origin"\n FROM\n "core_projectcollaborator" C1\n INNER JOIN "core_project" P1 ON (P1."id" = C1."project_id")\n INNER JOIN "core_user" U1 ON (P1."owner_id" = U1."id")\n ),\n project_collaborator_team AS (\n SELECT\n 5 AS rank,\n C1."project_id",\n TM1."member_id" AS "user_id",\n C1."role" AS "name",\n \'team_member\' AS "origin"\n FROM\n "core_projectcollaborator" C1\n INNER JOIN "core_user" U1 ON (C1."collaborator_id" = U1."id")\n INNER JOIN "core_team" T1 ON (U1."id" = T1."user_ptr_id")\n INNER JOIN "core_teammember" TM1 ON (T1."user_ptr_id" = TM1."team_id")\n INNER JOIN "core_project" P1 ON (P1."id" = C1."project_id")\n ),\n public_project AS (\n SELECT\n 6 AS rank,\n P1."id" AS "project_id",\n U1."id" AS "user_id",\n \'reader\' AS "name",\n \'public\' AS "origin"\n FROM\n "core_project" P1\n CROSS JOIN "core_user" U1\n WHERE\n is_public = TRUE\n )\n SELECT DISTINCT ON(project_id, user_id)\n nextval(\'projects_with_roles_vw_seq\') id,\n R1.*\n FROM (\n SELECT * FROM project_owner\n UNION\n SELECT * FROM organization_owner\n UNION\n SELECT * FROM organization_admin\n UNION\n SELECT * FROM project_collaborator\n UNION\n SELECT * FROM project_collaborator_team\n UNION\n SELECT * FROM public_project\n ) R1\n ORDER BY project_id, user_id, rank\n ', + ), + migrate_sql.operations.AlterSQL( + name="projects_with_roles_vw", + sql='\n CREATE OR REPLACE VIEW projects_with_roles_vw AS\n\n WITH project_owner AS (\n SELECT\n 1 AS rank,\n P1."id" AS "project_id",\n P1."owner_id" AS "user_id",\n \'admin\' AS "name",\n FALSE AS "is_incognito",\n \'project_owner\' AS "origin"\n FROM\n "core_project" P1\n INNER JOIN "core_user" U1 ON (P1."owner_id" = U1."id")\n WHERE\n U1."type" = 1\n ),\n organization_owner AS (\n SELECT\n 2 AS rank,\n P1."id" AS "project_id",\n O1."organization_owner_id" AS "user_id",\n \'admin\' AS "name",\n FALSE AS "is_incognito",\n \'organization_owner\' AS "origin"\n FROM\n "core_organization" O1\n INNER JOIN "core_project" P1 ON (P1."owner_id" = O1."user_ptr_id")\n ),\n organization_admin AS (\n SELECT\n 3 AS rank,\n P1."id" AS "project_id",\n OM1."member_id" AS "user_id",\n \'admin\' AS "name",\n FALSE AS "is_incognito",\n \'organization_admin\' AS "origin"\n FROM\n "core_organizationmember" OM1\n INNER JOIN "core_project" P1 ON (P1."owner_id" = OM1."organization_id")\n WHERE\n (\n OM1."role" = \'admin\'\n )\n ),\n project_collaborator AS (\n SELECT\n 4 AS rank,\n C1."project_id",\n C1."collaborator_id" AS "user_id",\n C1."role" AS "name",\n C1."is_incognito" AS "is_incognito",\n \'collaborator\' AS "origin"\n FROM\n "core_projectcollaborator" C1\n INNER JOIN "core_project" P1 ON (P1."id" = C1."project_id")\n INNER JOIN "core_user" U1 ON (P1."owner_id" = U1."id")\n ),\n project_collaborator_team AS (\n SELECT\n 5 AS rank,\n C1."project_id",\n TM1."member_id" AS "user_id",\n C1."role" AS "name",\n C1."is_incognito" AS "is_incognito",\n \'team_member\' AS "origin"\n FROM\n "core_projectcollaborator" C1\n INNER JOIN "core_user" U1 ON (C1."collaborator_id" = U1."id")\n INNER JOIN "core_team" T1 ON (U1."id" = T1."user_ptr_id")\n INNER JOIN "core_teammember" TM1 ON (T1."user_ptr_id" = TM1."team_id")\n INNER JOIN "core_project" P1 ON (P1."id" = C1."project_id")\n ),\n public_project AS (\n SELECT\n 6 AS rank,\n P1."id" AS "project_id",\n U1."id" AS "user_id",\n \'reader\' AS "name",\n FALSE AS "is_incognito",\n \'public\' AS "origin"\n FROM\n "core_project" P1\n CROSS JOIN "core_user" U1\n WHERE\n is_public = TRUE\n )\n SELECT DISTINCT ON(project_id, user_id)\n nextval(\'projects_with_roles_vw_seq\') id,\n R1.*\n FROM (\n SELECT * FROM project_owner\n UNION\n SELECT * FROM organization_owner\n UNION\n SELECT * FROM organization_admin\n UNION\n SELECT * FROM project_collaborator\n UNION\n SELECT * FROM project_collaborator_team\n UNION\n SELECT * FROM public_project\n ) R1\n ORDER BY project_id, user_id, rank\n ', + reverse_sql="\n DROP VIEW projects_with_roles_vw;\n ", + ), + ] diff --git a/docker-app/qfieldcloud/core/models.py b/docker-app/qfieldcloud/core/models.py index d14e5cfe5..ea7fdc888 100644 --- a/docker-app/qfieldcloud/core/models.py +++ b/docker-app/qfieldcloud/core/models.py @@ -1,4 +1,4 @@ -import contextlib +import logging import os import secrets import string @@ -9,7 +9,6 @@ import django_cryptography.fields import qfieldcloud.core.utils2.storage -from auditlog.registry import auditlog from deprecated import deprecated from django.contrib.auth.models import AbstractUser from django.contrib.auth.models import UserManager as DjangoUserManager @@ -31,6 +30,8 @@ # http://springmeblog.com/2018/how-to-implement-multiple-user-types-with-django/ +logger = logging.getLogger(__name__) + class PersonQueryset(models.QuerySet): """Adds for_project(user) method to the user's querysets, allowing to filter only users part of a project. @@ -313,8 +314,7 @@ def delete(self, *args, **kwargs): if self.type != User.Type.TEAM: qfieldcloud.core.utils2.storage.delete_user_avatar(self) - with no_audits([User, UserAccount, Project]): - super().delete(*args, **kwargs) + super().delete(*args, **kwargs) class Meta: base_manager_name = "objects" @@ -426,57 +426,49 @@ def avatar_url(self): return None @property - @deprecated( - "Use `UserAccount().active_subscription.active_storage_total_mb` instead." - ) - def storage_quota_total_mb(self) -> float: - """Returns the storage quota left in MB (quota from account and packages minus storage of all owned projects)""" - return ( - self.active_subscription.plan.storage_mb - + self.active_subscription.active_storage_package_mb - ) - - @property - @deprecated("Use `UserAccount().storage_used_mb` instead.") - def storage_quota_used_mb(self) -> float: - return self.storage_used_mb - - @property - @deprecated("Use `UserAccount().storage_free_mb` instead.") - def storage_quota_left_mb(self) -> float: - return self.storage_free_mb - - @property - @deprecated("Use `UserAccount().storage_used_ratio` instead") - def storage_quota_used_perc(self) -> float: - return self.storage_used_ratio - - @property - @deprecated("Use `UserAccount().storage_free_ratio` instead") - def storage_quota_left_perc(self) -> float: - return self.storage_free_ratio - - @property + @deprecated("Use `UserAccount().storage_used_bytes` instead") + # TODO delete this method after refactoring tests so it's no longer used there def storage_used_mb(self) -> float: """Returns the storage used in MB""" + return self.storage_used_bytes / 1000 / 1000 + + @property + def storage_used_bytes(self) -> float: + """Returns the storage used in bytes""" used_quota = ( - self.user.projects.aggregate(sum_mb=Sum("storage_size_mb"))["sum_mb"] or 0 + self.user.projects.aggregate(sum_bytes=Sum("file_storage_bytes"))[ + "sum_bytes" + ] + # if there are no projects, the value will be `None` + or 0 ) return used_quota @property + @deprecated("Use `UserAccount().storage_free_bytes` instead") + # TODO delete this method after refactoring tests so it's no longer used there def storage_free_mb(self) -> float: """Returns the storage quota left in MB (quota from account and packages minus storage of all owned projects)""" - return self.active_subscription.active_storage_total_mb - self.storage_used_mb + return self.storage_free_bytes / 1000 / 1000 + + @property + def storage_free_bytes(self) -> float: + """Returns the storage quota left in bytes (quota from account and packages minus storage of all owned projects)""" + + return ( + self.active_subscription.active_storage_total_bytes + - self.storage_used_bytes + ) @property def storage_used_ratio(self) -> float: """Returns the storage used in fraction of the total storage""" - if self.active_subscription.active_storage_total_mb > 0: + if self.active_subscription.active_storage_total_bytes > 0: return min( - self.storage_used_mb / self.active_subscription.active_storage_total_mb, + self.storage_used_bytes + / self.active_subscription.active_storage_total_bytes, 1, ) else: @@ -508,6 +500,9 @@ def has_premium_support(self) -> bool: return False + def __str__(self) -> str: + return f"{self.user.username_with_full_name} ({self.__class__.__name__})" + class Geodb(models.Model): def random_string(): @@ -679,8 +674,8 @@ def active_users(self, period_since: datetime, period_until: datetime): ) return Person.objects.filter( - Q(id__in=users_with_delta) | Q(id__in=users_with_jobs) - ) + is_staff=False, + ).filter(Q(id__in=users_with_delta) | Q(id__in=users_with_jobs)) def save(self, *args, **kwargs): self.type = User.Type.ORGANIZATION @@ -927,6 +922,7 @@ def for_user(self, user: "User", skip_invalid: bool = False): user_role_is_valid=Case( When(is_valid_user_role_q, then=True), default=False ), + user_role_is_incognito=F("user_roles__is_incognito"), ) ) @@ -1005,7 +1001,7 @@ class Meta: # These cache stats of the S3 storage. These can be out of sync, and should be # refreshed whenever retrieving/uploading files by passing `project.save(recompute_storage=True)` - storage_size_mb = models.FloatField(default=0) + file_storage_bytes = models.PositiveBigIntegerField(default=0) # NOTE we can track only the file based layers, WFS, WMS, PostGIS etc are impossible to track data_last_updated_at = models.DateTimeField(blank=True, null=True) @@ -1178,9 +1174,14 @@ def status_code(self) -> StatusCode: @property def storage_size_perc(self) -> float: - return ( - self.storage_size_mb / self.owner.useraccount.storage_quota_total_mb * 100 - ) + if self.owner.useraccount.active_subscription.active_storage_total_bytes > 0: + return ( + self.file_storage_bytes + / self.owner.useraccount.active_subscription.active_storage_total_bytes + * 100 + ) + else: + return 100 @property def direct_collaborators(self): @@ -1189,8 +1190,14 @@ def direct_collaborators(self): else: exclude_pks = [self.owner_id] - return self.collaborators.filter(collaborator__type=User.Type.PERSON,).exclude( - collaborator_id__in=exclude_pks, + return ( + self.collaborators.skip_incognito() + .filter( + collaborator__type=User.Type.PERSON, + ) + .exclude( + collaborator_id__in=exclude_pks, + ) ) def delete(self, *args, **kwargs): @@ -1199,14 +1206,20 @@ def delete(self, *args, **kwargs): super().delete(*args, **kwargs) def save(self, recompute_storage=False, *args, **kwargs): + logger.info(f"Saving project {self}...") + if recompute_storage: - self.storage_size_mb = utils.get_s3_project_size(self.id) + self.file_storage_bytes = ( + qfieldcloud.core.utils2.storage.get_project_file_storage_in_bytes( + self.id + ) + ) super().save(*args, **kwargs) class ProjectCollaboratorQueryset(models.QuerySet): def validated(self, skip_invalid=False): - """Annotates the queryset with `is_valid` and by default filters out all invalid memberships. + """Annotates the queryset with `is_valid` and by default filters out all invalid memberships if `skip_invalid` is set to True. A membership to a private project is valid when the owning user plan has a `max_premium_collaborators_per_private_project` >= of the total count of project collaborators. @@ -1214,11 +1227,17 @@ def validated(self, skip_invalid=False): Args: skip_invalid: if true, invalid rows are removed""" count = Count( - "project__collaborators", filter=Q(collaborator__type=User.Type.PERSON) + "project__collaborators", + filter=Q( + collaborator__type=User.Type.PERSON, + # incognito users should never be counted + is_incognito=False, + ), ) # Build the conditions with Q objects is_public_q = Q(project__is_public=True) + is_team_collaborator = Q(collaborator__type=User.Type.TEAM) # max_premium_collaborators_per_private_project_q = active_subscription_q & ( max_premium_collaborators_per_private_project_q = Q( project__owner__useraccount__current_subscription__plan__max_premium_collaborators_per_private_project=V( @@ -1230,7 +1249,9 @@ def validated(self, skip_invalid=False): # Assemble the condition is_valid_collaborator = ( - is_public_q | max_premium_collaborators_per_private_project_q + is_public_q + | max_premium_collaborators_per_private_project_q + | is_team_collaborator ) # Annotate the queryset @@ -1244,6 +1265,9 @@ def validated(self, skip_invalid=False): return qs + def skip_incognito(self): + return self.filter(is_incognito=False) + class ProjectCollaborator(models.Model): class Roles(models.TextChoices): @@ -1275,6 +1299,38 @@ class Meta: ) role = models.CharField(max_length=10, choices=Roles.choices, default=Roles.READER) + # whether the collaborator is incognito, e.g. shown in the UI and billed + is_incognito = models.BooleanField( + default=False, + help_text=_( + "If a collaborator is marked as incognito, they will work as normal, but will not be listed in the UI or accounted in the subscription as active users. Used to add OPENGIS.ch support members to projects." + ), + ) + + # created by + created_by = models.ForeignKey( + Person, + on_delete=models.SET_NULL, + related_name="+", + null=True, + blank=True, + ) + + # created at + created_at = models.DateTimeField(auto_now_add=True) + + # created by + updated_by = models.ForeignKey( + Person, + on_delete=models.SET_NULL, + related_name="+", + null=True, + blank=True, + ) + + # updated at + updated_at = models.DateTimeField(auto_now=True) + def __str__(self): return self.project.name + ": " + self.collaborator.username @@ -1321,6 +1377,7 @@ class ProjectRolesView(models.Model): origin = models.CharField( max_length=100, choices=ProjectQueryset.RoleOrigins.choices ) + is_incognito = models.BooleanField() class Meta: db_table = "projects_with_roles_vw" @@ -1581,66 +1638,3 @@ class Meta: fields=["project", "name"], name="secret_project_name_uniq" ) ] - - -audited_models = [ - (User, {"exclude_fields": ["last_login", "updated_at"]}), - (UserAccount, {}), - (Organization, {}), - (OrganizationMember, {}), - (Team, {}), - (TeamMember, {}), - ( - Project, - { - "include_fields": [ - "id", - "name", - "description", - "owner", - "is_public", - "owner", - "created_at", - ], - }, - ), - (ProjectCollaborator, {}), - ( - Delta, - { - "include_fields": [ - "id", - "deltafile_id", - "project", - "content", - "last_status", - "created_by", - ], - }, - ), - (Secret, {"exclude_fields": ["value"]}), -] - - -def register_model_audits(subset: List[models.Model] = []) -> None: - for model, kwargs in audited_models: - if subset and model not in subset: - continue - auditlog.register(model, **kwargs) - - -def deregister_model_audits(subset: List[models.Model] = []) -> None: - for model, _kwargs in audited_models: - if subset and model not in subset: - continue - - auditlog.unregister(model) - - -@contextlib.contextmanager -def no_audits(subset: List[models.Model] = []): - try: - deregister_model_audits(subset) - yield - finally: - register_model_audits(subset) diff --git a/docker-app/qfieldcloud/core/sql_config.py b/docker-app/qfieldcloud/core/sql_config.py index 6c98da9f1..e2dd39501 100644 --- a/docker-app/qfieldcloud/core/sql_config.py +++ b/docker-app/qfieldcloud/core/sql_config.py @@ -21,6 +21,7 @@ P1."id" AS "project_id", P1."owner_id" AS "user_id", 'admin' AS "name", + FALSE AS "is_incognito", 'project_owner' AS "origin" FROM "core_project" P1 @@ -34,6 +35,7 @@ P1."id" AS "project_id", O1."organization_owner_id" AS "user_id", 'admin' AS "name", + FALSE AS "is_incognito", 'organization_owner' AS "origin" FROM "core_organization" O1 @@ -45,6 +47,7 @@ P1."id" AS "project_id", OM1."member_id" AS "user_id", 'admin' AS "name", + FALSE AS "is_incognito", 'organization_admin' AS "origin" FROM "core_organizationmember" OM1 @@ -60,6 +63,7 @@ C1."project_id", C1."collaborator_id" AS "user_id", C1."role" AS "name", + C1."is_incognito" AS "is_incognito", 'collaborator' AS "origin" FROM "core_projectcollaborator" C1 @@ -72,6 +76,7 @@ C1."project_id", TM1."member_id" AS "user_id", C1."role" AS "name", + C1."is_incognito" AS "is_incognito", 'team_member' AS "origin" FROM "core_projectcollaborator" C1 @@ -86,6 +91,7 @@ P1."id" AS "project_id", U1."id" AS "user_id", 'reader' AS "name", + FALSE AS "is_incognito", 'public' AS "origin" FROM "core_project" P1 diff --git a/docker-app/qfieldcloud/core/tests/test_admin.py b/docker-app/qfieldcloud/core/tests/test_admin.py index 5f5052209..355317442 100644 --- a/docker-app/qfieldcloud/core/tests/test_admin.py +++ b/docker-app/qfieldcloud/core/tests/test_admin.py @@ -68,7 +68,9 @@ def test_admin_opens(self): "/admin/core/delta/add/", "/admin/core/job/add/", "/admin/axes/accessattempt/add/", + "/admin/axes/accessfailurelog/add/", "/admin/axes/accesslog/add/", + "/admin/auditlog/logentry/add/", ) # TODO make tests pass for these sortable URLs skip_sort_urls = ("/admin/django_cron/cronjoblog/?o=4",) diff --git a/docker-app/qfieldcloud/core/tests/test_organization.py b/docker-app/qfieldcloud/core/tests/test_organization.py index ab7a50248..0f1a48f22 100644 --- a/docker-app/qfieldcloud/core/tests/test_organization.py +++ b/docker-app/qfieldcloud/core/tests/test_organization.py @@ -37,6 +37,12 @@ def setUp(self): self.user3 = Person.objects.create_user(username="user3", password="abc123") self.token3 = AuthToken.objects.get_or_create(user=self.user3)[0] + # Create a staff user + self.user4 = Person.objects.create_user( + username="user4", password="abc123", is_staff=True + ) + self.token4 = AuthToken.objects.get_or_create(user=self.user4)[0] + # Create an organization self.organization1 = Organization.objects.create( username="organization1", @@ -184,6 +190,11 @@ def test_active_users_count(self): member=self.user3, role=OrganizationMember.Roles.MEMBER, ) + OrganizationMember.objects.create( + organization=self.organization1, + member=self.user4, + role=OrganizationMember.Roles.MEMBER, + ) # Create a project owned by the organization project1 = Project.objects.create(name="p1", owner=self.organization1) @@ -229,14 +240,22 @@ def _active_users_count(base_date=None): project=project1, created_by=self.user3, ) - # There is 2 billable user + # There are 2 billable users self.assertEqual(_active_users_count(), 2) # User 2 leaves the organization OrganizationMember.objects.filter(member=self.user3).delete() - # There are strill 2 billable user + # There are still 2 billable users self.assertEqual(_active_users_count(), 2) # Report at a different time is empty self.assertEqual(_active_users_count(now() + timedelta(days=365)), 0) + + # User 3 creates a job + Job.objects.create( + project=project1, + created_by=self.user3, + ) + # There are still 2 billable users, because self.user3 is staff + self.assertEqual(_active_users_count(), 2) diff --git a/docker-app/qfieldcloud/core/tests/test_project.py b/docker-app/qfieldcloud/core/tests/test_project.py index 6ac776dfc..628edd25e 100644 --- a/docker-app/qfieldcloud/core/tests/test_project.py +++ b/docker-app/qfieldcloud/core/tests/test_project.py @@ -404,6 +404,27 @@ def test_add_project_collaborator_without_being_org_member(self): project=p1, collaborator=u2, role=ProjectCollaborator.Roles.MANAGER ) + def test_direct_collaborators(self): + u1 = Person.objects.create(username="u1") + u2 = Person.objects.create(username="u2") + o1 = Organization.objects.create(username="o1", organization_owner=u1) + p1 = Project.objects.create(name="p1", owner=o1, is_public=False) + + OrganizationMember.objects.create(organization=o1, member=u2) + c1 = ProjectCollaborator.objects.create( + project=p1, + collaborator=u2, + role=ProjectCollaborator.Roles.MANAGER, + is_incognito=False, + ) + + self.assertEqual(len(p1.direct_collaborators), 1) + + c1.is_incognito = True + c1.save() + + self.assertEqual(len(p1.direct_collaborators), 0) + def test_add_project_collaborator_and_being_org_member(self): u1 = Person.objects.create(username="u1") u2 = Person.objects.create(username="u2") diff --git a/docker-app/qfieldcloud/core/tests/test_qgis_file.py b/docker-app/qfieldcloud/core/tests/test_qgis_file.py index d22d9d054..d43af5fc6 100644 --- a/docker-app/qfieldcloud/core/tests/test_qgis_file.py +++ b/docker-app/qfieldcloud/core/tests/test_qgis_file.py @@ -3,7 +3,6 @@ import tempfile import time from pathlib import PurePath -from unittest import skip from django.core.management import call_command from django.http import FileResponse @@ -130,7 +129,7 @@ def test_push_download_file_with_path(self): open(testdata_path("file.txt"), "rb").read(), ) - def test_push_list_file(self): + def test_upload_and_list_file(self): self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key) self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0) @@ -178,7 +177,7 @@ def test_push_list_file(self): "fcc85fb502bd772aa675a0263b5fa665bccd5d8d93349d1dbc9f0f6394dd37b9", ) - def test_push_list_file_with_space_in_name(self): + def test_upload_and_list_file_with_space_in_name(self): self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key) self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0) @@ -208,13 +207,14 @@ def test_push_list_file_with_space_in_name(self): self.assertEqual(json[0]["name"], "aaa bbb/project qgis 1.2.qgs") - def test_push_list_file_versions(self): + def test_upload_and_list_file_versions(self): self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key) - self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0) - self.assertEqual( - Project.objects.get(pk=self.project1.pk).project_filename, None - ) + project = Project.objects.get(pk=self.project1.pk) + + self.assertEqual(project.files_count, 0) + self.assertEqual(project.file_storage_bytes, 0) + self.assertIsNone(project.project_filename) file_path = testdata_path("file.txt") # Push a file @@ -223,8 +223,12 @@ def test_push_list_file_versions(self): {"file": open(file_path, "rb")}, format="multipart", ) + project = Project.objects.get(pk=self.project1.pk) + self.assertTrue(status.is_success(response.status_code)) - self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1) + self.assertEqual(project.files_count, 1) + self.assertEqual(project.file_storage_bytes, 13) + self.assertIsNone(project.project_filename) # Wait 2 seconds to be sure the file timestamps are different time.sleep(2) @@ -237,8 +241,12 @@ def test_push_list_file_versions(self): {"file": open(file_path, "rb")}, format="multipart", ) + project = Project.objects.get(pk=self.project1.pk) + self.assertTrue(status.is_success(response.status_code)) - self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1) + self.assertEqual(project.files_count, 1) + self.assertEqual(project.file_storage_bytes, 26) + self.assertIsNone(project.project_filename) # List files response = self.client.get("/api/v1/files/{}/".format(self.project1.id)) @@ -332,10 +340,11 @@ def test_push_download_specific_version_file(self): def test_push_delete_file(self): self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key) - self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0) - self.assertEqual( - Project.objects.get(pk=self.project1.pk).project_filename, None - ) + project = Project.objects.get(pk=self.project1.pk) + + self.assertEqual(project.files_count, 0) + self.assertEqual(project.file_storage_bytes, 0) + self.assertIsNone(project.project_filename) file_path = testdata_path("file.txt") # Push a file @@ -344,8 +353,12 @@ def test_push_delete_file(self): {"file": open(file_path, "rb")}, format="multipart", ) + project = Project.objects.get(pk=self.project1.pk) + self.assertTrue(status.is_success(response.status_code)) - self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1) + self.assertEqual(project.files_count, 1) + self.assertEqual(project.file_storage_bytes, 13) + self.assertIsNone(project.project_filename) file_path = testdata_path("file2.txt") # Push a second file @@ -354,8 +367,12 @@ def test_push_delete_file(self): {"file": open(file_path, "rb")}, format="multipart", ) + project = Project.objects.get(pk=self.project1.pk) + self.assertTrue(status.is_success(response.status_code)) - self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 2) + self.assertEqual(project.files_count, 2) + self.assertEqual(project.file_storage_bytes, 26) + self.assertIsNone(project.project_filename) # List files response = self.client.get("/api/v1/files/{}/".format(self.project1.id)) @@ -366,8 +383,12 @@ def test_push_delete_file(self): response = self.client.delete( "/api/v1/files/{}/aaa/file.txt/".format(self.project1.id) ) + project = Project.objects.get(pk=self.project1.pk) + self.assertTrue(status.is_success(response.status_code)) - self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1) + self.assertEqual(project.files_count, 1) + self.assertEqual(project.file_storage_bytes, 13) + self.assertIsNone(project.project_filename) # List files response = self.client.get("/api/v1/files/{}/".format(self.project1.id)) @@ -464,7 +485,7 @@ def test_upload_1mb_file(self): big_file = tempfile.NamedTemporaryFile() with open(big_file.name, "wb") as bf: - bf.truncate(1024 * 1024 * 1) + bf.truncate(1000 * 1000 * 1) # Push the file response = self.client.post( @@ -472,8 +493,11 @@ def test_upload_1mb_file(self): data={"file": open(big_file.name, "rb")}, format="multipart", ) + project = Project.objects.get(pk=self.project1.pk) + self.assertTrue(status.is_success(response.status_code)) - self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1) + self.assertEqual(project.files_count, 1) + self.assertEqual(project.file_storage_bytes, 1000000) # List files response = self.client.get("/api/v1/files/{}/".format(self.project1.id)) @@ -481,8 +505,7 @@ def test_upload_1mb_file(self): self.assertTrue(status.is_success(response.status_code)) self.assertEqual(len(response.json()), 1) self.assertEqual("bigfile.big", response.json()[0]["name"]) - self.assertGreater(response.json()[0]["size"], 1000000) - self.assertLess(response.json()[0]["size"], 1100000) + self.assertEqual(response.json()[0]["size"], 1000000) def test_upload_10mb_file(self): self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key) @@ -494,7 +517,7 @@ def test_upload_10mb_file(self): big_file = tempfile.NamedTemporaryFile() with open(big_file.name, "wb") as bf: - bf.truncate(1024 * 1024 * 10) + bf.truncate(1000 * 1000 * 10) # Push the file response = self.client.post( @@ -502,17 +525,18 @@ def test_upload_10mb_file(self): data={"file": open(big_file.name, "rb")}, format="multipart", ) - self.assertTrue(status.is_success(response.status_code)) - self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1) + project = Project.objects.get(pk=self.project1.pk) + self.assertTrue(status.is_success(response.status_code)) + self.assertEqual(project.files_count, 1) + self.assertEqual(project.file_storage_bytes, 10000000) # List files response = self.client.get("/api/v1/files/{}/".format(self.project1.id)) self.assertTrue(status.is_success(response.status_code)) self.assertEqual(len(response.json()), 1) self.assertEqual("bigfile.big", response.json()[0]["name"]) - self.assertGreater(response.json()[0]["size"], 10000000) - self.assertLess(response.json()[0]["size"], 11000000) + self.assertEqual(response.json()[0]["size"], 10000000) def test_purge_old_versions_command(self): """This tests manual purging of old versions with the management command""" @@ -560,9 +584,6 @@ def read_version(n): self.assertEqual(read_version(0), "v17") self.assertEqual(read_version(2), "v19") - @skip( - "Temporary disable the purge old versions, as we temporarily disabled purging old versions." - ) def test_purge_old_versions(self): """This tests automated purging of old versions when uploading files""" diff --git a/docker-app/qfieldcloud/core/utils.py b/docker-app/qfieldcloud/core/utils.py index d27a57c16..76b34e386 100644 --- a/docker-app/qfieldcloud/core/utils.py +++ b/docker-app/qfieldcloud/core/utils.py @@ -306,25 +306,6 @@ def get_deltafile_schema_validator() -> jsonschema.Draft7Validator: return jsonschema.Draft7Validator(schema_dict) -def get_s3_project_size(project_id: str) -> int: - """Return the size in MB of the project on the storage, including the - exported files and their versions""" - - bucket = get_s3_bucket() - - total_size = 0 - - files_prefix = f"projects/{project_id}/files/" - for version in bucket.object_versions.filter(Prefix=files_prefix): - total_size += version.size or 0 - - packages_prefix = f"projects/{project_id}/packages/" - for version in bucket.object_versions.filter(Prefix=packages_prefix): - total_size += version.size or 0 - - return round(total_size / (1000 * 1000), 3) - - def get_project_files(project_id: str, path: str = "") -> Iterable[S3Object]: """Returns a list of files and their versions. diff --git a/docker-app/qfieldcloud/core/utils2/projects.py b/docker-app/qfieldcloud/core/utils2/projects.py index 1190f837e..14e49d606 100644 --- a/docker-app/qfieldcloud/core/utils2/projects.py +++ b/docker-app/qfieldcloud/core/utils2/projects.py @@ -4,15 +4,18 @@ from django.utils.translation import gettext as _ from qfieldcloud.core import invitations_utils as invitation from qfieldcloud.core import permissions_utils as perms -from qfieldcloud.core.models import Person, Project, ProjectCollaborator, User +from qfieldcloud.core.models import Person, Project, ProjectCollaborator, Team, User -def create_collaborator(project: Project, user: User) -> Tuple[bool, str]: +def create_collaborator( + project: Project, user: User, created_by: Person +) -> Tuple[bool, str]: """Creates a new collaborator (qfieldcloud.core.ProjectCollaborator) if possible Args: project (Project): the project to add collaborator to user (User): the user to be added as collaborator + created_by (Person): the user that initiated the collaborator creation Returns: Tuple[bool, str]: success, message - whether the collaborator creation was success and explanation message of the outcome @@ -27,6 +30,8 @@ def create_collaborator(project: Project, user: User) -> Tuple[bool, str]: ProjectCollaborator.objects.create( project=project, collaborator=user, + created_by=created_by, + updated_by=created_by, ) success = True message = _('User "{}" has been invited to the project.').format(user.username) @@ -59,7 +64,9 @@ def create_collaborator_by_username_or_email( Tuple[bool, str]: success, message - whether the collaborator creation was success and explanation message of the outcome """ success, message = False, "" - users = list(Person.objects.filter(Q(username=username) | Q(email=username))) + users = list( + Person.objects.filter(Q(username=username) | Q(email=username)) + ) + list(Team.objects.filter(username=username, team_organization=project.owner)) if len(users) == 0: # No user found, if string is an email address, we try to send a link @@ -79,6 +86,6 @@ def create_collaborator_by_username_or_email( ).format(username), ) else: - success, message = create_collaborator(project, users[0]) + success, message = create_collaborator(project, users[0], created_by) return success, message diff --git a/docker-app/qfieldcloud/core/utils2/storage.py b/docker-app/qfieldcloud/core/utils2/storage.py index 1b6e2e395..9a5b56d1d 100644 --- a/docker-app/qfieldcloud/core/utils2/storage.py +++ b/docker-app/qfieldcloud/core/utils2/storage.py @@ -4,7 +4,7 @@ import os import re from pathlib import PurePath -from typing import IO, List, Set +from typing import IO, List, Optional, Set import qfieldcloud.core.models import qfieldcloud.core.utils @@ -13,6 +13,7 @@ from django.db import transaction from django.http import FileResponse, HttpRequest from django.http.response import HttpResponse, HttpResponseBase +from mypy_boto3_s3.type_defs import ObjectIdentifierTypeDef from qfieldcloud.core.utils2.audit import LogEntry, audit logger = logging.getLogger(__name__) @@ -132,7 +133,7 @@ def _delete_by_key_permanently(key: str): temp_objects = bucket.object_versions.filter( Prefix=key, ) - object_to_delete = [] + object_to_delete: List[ObjectIdentifierTypeDef] = [] for temp_object in temp_objects: # filter out objects that do not have the same key as the requested deletion key. if temp_object.key != key: @@ -176,7 +177,9 @@ def delete_version_permanently(version_obj: qfieldcloud.core.utils.S3ObjectVersi version_obj._data.delete() -def get_attachment_dir_prefix(project: "Project", filename: str) -> str: # noqa: F821 +def get_attachment_dir_prefix( + project: qfieldcloud.core.models.Project, filename: str +) -> str: # noqa: F821 """Returns the attachment dir where the file belongs to or empty string if it does not. Args: @@ -198,7 +201,7 @@ def file_response( key: str, presigned: bool = False, expires: int = 60, - version: str = None, + version: Optional[str] = None, as_attachment: bool = False, ) -> HttpResponseBase: url = "" @@ -259,7 +262,9 @@ def file_response( ) -def upload_user_avatar(user: "User", file: IO, mimetype: str) -> str: # noqa: F821 +def upload_user_avatar( + user: qfieldcloud.core.models.User, file: IO, mimetype: str +) -> str: # noqa: F821 """Uploads a picture as a user avatar. NOTE this function does NOT modify the `UserAccount.avatar_uri` field @@ -295,7 +300,7 @@ def upload_user_avatar(user: "User", file: IO, mimetype: str) -> str: # noqa: F return key -def delete_user_avatar(user: "User") -> None: # noqa: F821 +def delete_user_avatar(user: qfieldcloud.core.models.User) -> None: # noqa: F821 """Deletes the user's avatar file. NOTE this function does NOT modify the `UserAccount.avatar_uri` field @@ -310,14 +315,17 @@ def delete_user_avatar(user: "User") -> None: # noqa: F821 return # e.g. "users/suricactus/avatar.svg" - if not key or not re.match(r"^users/\w+/avatar\.(png|jpg|svg)$", key): + if not key or not re.match(r"^users/[\w-]+/avatar\.(png|jpg|svg)$", key): raise RuntimeError(f"Suspicious S3 deletion of user avatar {key=}") _delete_by_key_permanently(key) def upload_project_thumbail( - project: "Project", file: IO, mimetype: str, filename: str # noqa: F821 + project: qfieldcloud.core.models.Project, + file: IO, + mimetype: str, + filename: str, # noqa: F821 ) -> str: """Uploads a picture as a project thumbnail. @@ -357,7 +365,9 @@ def upload_project_thumbail( return key -def delete_project_thumbnail(project: "Project") -> None: # noqa: F821 +def delete_project_thumbnail( + project: qfieldcloud.core.models.Project, +) -> None: # noqa: F821 """Delete a picture as a project thumbnail. NOTE this function does NOT modify the `Project.thumbnail_uri` field @@ -379,7 +389,9 @@ def delete_project_thumbnail(project: "Project") -> None: # noqa: F821 _delete_by_key_permanently(key) -def purge_old_file_versions(project: "Project") -> None: # noqa: F821 +def purge_old_file_versions( + project: qfieldcloud.core.models.Project, +) -> None: # noqa: F821 """ Deletes old versions of all files in the given project. Will keep __3__ versions for COMMUNITY user accounts, and __10__ versions for PRO user @@ -440,7 +452,7 @@ def upload_file(file: IO, key: str): def upload_project_file( - project: "Project", file: IO, filename: str # noqa: F821 + project: qfieldcloud.core.models.Project, file: IO, filename: str # noqa: F821 ) -> str: key = f"projects/{project.id}/files/{filename}" bucket = qfieldcloud.core.utils.get_s3_bucket() @@ -462,7 +474,11 @@ def delete_all_project_files_permanently(project_id: str) -> None: _delete_by_prefix_versioned(prefix) -def delete_project_file_permanently(project: "Project", filename: str): # noqa: F821 +def delete_project_file_permanently( + project: qfieldcloud.core.models.Project, filename: str +): # noqa: F821 + logger.info(f"Requested delete (permanent) of project file {filename=}") + file = qfieldcloud.core.utils.get_project_file_with_versions(project.id, filename) if not file: @@ -470,28 +486,40 @@ def delete_project_file_permanently(project: "Project", filename: str): # noqa: f"No file with such name in the given project found {filename=}" ) + if not re.match(r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/.+$", file.latest.key): + raise RuntimeError(f"Suspicious S3 file deletion {file.latest.key=}") + + # NOTE the file operations depend on HTTP calls to the S3 storage and they might fail, + # we need to choose source of truth between DB and S3. + # For now the source of truth is what is on the S3 storage, + # because we do most of our file operations directly by calling the S3 API. + # 1. S3 storage modification. If it fails, it will cancel the transaction + # and do not update anything in the database. + # We assume S3 storage is transactional, while it might not be true. + # 2. DB modification. If it fails, the states betewen DB and S3 mismatch, + # but can be easyly synced from the S3 to DB with a manual script. with transaction.atomic(): + _delete_by_key_permanently(file.latest.key) + + update_fields = ["file_storage_bytes"] + if qfieldcloud.core.utils.is_qgis_project_file(filename): + update_fields.append("project_filename") project.project_filename = None - project.save(recompute_storage=True, update_fields=["project_filename"]) - # NOTE auditing the file deletion in the transation might be costly, but guarantees the audit + project.file_storage_bytes -= sum([v.size for v in file.versions]) + project.save(update_fields=update_fields) + + # NOTE force audits to be required when deleting files audit( project, LogEntry.Action.DELETE, changes={f"{filename} ALL": [file.latest.e_tag, None]}, ) - if not re.match( - r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/.+$", file.latest.key - ): - raise RuntimeError(f"Suspicious S3 file deletion {file.latest.key=}") - - _delete_by_key_permanently(file.latest.key) - def delete_project_file_version_permanently( - project: "Project", # noqa: F821 + project: qfieldcloud.core.models.Project, # noqa: F821 filename: str, version_id: str, include_older: bool = False, @@ -599,3 +627,25 @@ def delete_stored_package(project_id: str, package_id: str) -> None: ) _delete_by_prefix_permanently(prefix) + + +def get_project_file_storage_in_bytes(project_id: str) -> int: + """Calculates the project files storage in bytes, including their versions. + + WARNING This function can be quite slow on projects with thousands of files. + """ + bucket = qfieldcloud.core.utils.get_s3_bucket() + total_bytes = 0 + prefix = f"projects/{project_id}/files/" + + logger.info(f"Project file storage size requrested for {project_id=}") + + if not re.match(r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/files/$", prefix): + raise RuntimeError( + f"Suspicious S3 calculation of all project files with {prefix=}" + ) + + for version in bucket.object_versions.filter(Prefix=prefix): + total_bytes += version.size or 0 + + return total_bytes diff --git a/docker-app/qfieldcloud/core/views/files_views.py b/docker-app/qfieldcloud/core/views/files_views.py index 1c0bb514e..be5a84fc1 100644 --- a/docker-app/qfieldcloud/core/views/files_views.py +++ b/docker-app/qfieldcloud/core/views/files_views.py @@ -7,7 +7,10 @@ from qfieldcloud.core.models import Job, ProcessProjectfileJob, Project from qfieldcloud.core.utils import S3ObjectVersion, get_project_file_with_versions from qfieldcloud.core.utils2.audit import LogEntry, audit -from qfieldcloud.core.utils2.storage import get_attachment_dir_prefix +from qfieldcloud.core.utils2.storage import ( + get_attachment_dir_prefix, + purge_old_file_versions, +) from rest_framework import permissions, status, views from rest_framework.parsers import MultiPartParser from rest_framework.response import Response @@ -127,6 +130,7 @@ def get(self, request, projectid, filename): as_attachment=True, ) + # TODO refactor this function by moving the actual upload and Project model updates to library function outside the view def post(self, request, projectid, filename, format=None): project = Project.objects.get(id=projectid) @@ -146,12 +150,12 @@ def post(self, request, projectid, filename, format=None): request_file = request.FILES.get("file") - file_size_mb = request_file.size / 1024 / 1024 - quota_left_mb = project.owner.useraccount.storage_free_mb + file_size_bytes = request_file.size + quota_left_bytes = project.owner.useraccount.storage_free_bytes - if file_size_mb > quota_left_mb: + if file_size_bytes > quota_left_bytes: raise exceptions.QuotaError( - f"Requiring {file_size_mb}MB of storage but only {quota_left_mb}MB available." + f"Requiring {file_size_bytes} bytes of storage but only {quota_left_bytes} bytes available." ) old_object = get_project_file_with_versions(project.id, filename) @@ -173,7 +177,7 @@ def post(self, request, projectid, filename, format=None): # project and update it now, it guarantees there will be no other file upload editing # the same project row. project = Project.objects.select_for_update().get(id=projectid) - update_fields = ["data_last_updated_at"] + update_fields = ["data_last_updated_at", "file_storage_bytes"] if get_attachment_dir_prefix(project, filename) == "" and ( is_qgis_project_file or project.project_filename is not None @@ -198,6 +202,8 @@ def post(self, request, projectid, filename, format=None): ) project.data_last_updated_at = timezone.now() + # NOTE just incrementing the fils_storage_bytes when uploading might make the database out of sync if a files is uploaded/deleted bypassing this function + project.file_storage_bytes += request_file.size project.save(update_fields=update_fields) if old_object: @@ -214,7 +220,7 @@ def post(self, request, projectid, filename, format=None): ) # Delete the old file versions - # purge_old_file_versions(project) + purge_old_file_versions(project) return Response(status=status.HTTP_201_CREATED) diff --git a/docker-app/qfieldcloud/core/views/projects_views.py b/docker-app/qfieldcloud/core/views/projects_views.py index e0de33458..97b8098be 100644 --- a/docker-app/qfieldcloud/core/views/projects_views.py +++ b/docker-app/qfieldcloud/core/views/projects_views.py @@ -141,7 +141,7 @@ def perform_update(self, serializer): # Owner has changed, we must ensure he has enough quota for that # (in this transaction, the project is his already, so we just need to # check his quota) - if new_owner.useraccount.storage_free_mb < 0: + if new_owner.useraccount.storage_free_bytes < 0: # If not, we rollback the transaction # (don't give away numbers in message as it's potentially private) raise exceptions.QuotaError( diff --git a/docker-app/qfieldcloud/settings.py b/docker-app/qfieldcloud/settings.py index 6f138317e..50735cf93 100644 --- a/docker-app/qfieldcloud/settings.py +++ b/docker-app/qfieldcloud/settings.py @@ -16,6 +16,9 @@ import sentry_sdk from sentry_sdk.integrations.django import DjangoIntegration +# QFieldCloud specific configuration +QFIELDCLOUD_HOST = os.environ["QFIELDCLOUD_HOST"] + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -127,7 +130,7 @@ # "qfieldcloud.core.cron.DeleteExpiredInvitationsJob", "qfieldcloud.core.cron.ResendFailedInvitationsJob", "qfieldcloud.core.cron.SetTerminatedWorkersToFinalStatusJob", - # "qfieldcloud.core.cron.DeleteObsoleteProjectPackagesJob", + "qfieldcloud.core.cron.DeleteObsoleteProjectPackagesJob", ] ROOT_URLCONF = "qfieldcloud.urls" @@ -253,17 +256,49 @@ LOGIN_URL = "account_login" - -sentry_sdk.init( - dsn=os.environ.get("SENTRY_DSN", ""), - integrations=[DjangoIntegration()], - # Define how many random events are sent for performance monitoring - sample_rate=0.05, - server_name=os.environ.get("QFIELDCLOUD_HOST"), - # If you wish to associate users to errors (assuming you are using - # django.contrib.auth) you may enable sending PII data. - send_default_pii=True, -) +# Sentry configuration +SENTRY_DSN = os.environ.get("SENTRY_DSN", "") +if SENTRY_DSN: + SENTRY_SAMPLE_RATE = float(os.environ.get("SENTRY_SAMPLE_RATE", 1)) + + def before_send(event, hint): + from qfieldcloud.core.exceptions import ProjectAlreadyExistsError + from rest_framework.exceptions import ValidationError + + ignored_exceptions = ( + ValidationError, + ProjectAlreadyExistsError, + ) + + if "exc_info" in hint: + + exc_class, _exc_object, _exc_tb = hint["exc_info"] + + # Skip sending errors + if issubclass(exc_class, ignored_exceptions): + return None + + return event + + sentry_sdk.init( + dsn=SENTRY_DSN, + integrations=[DjangoIntegration()], + server_name=QFIELDCLOUD_HOST, + # + # Sentry sample rate between 0 and 1. Read more on https://docs.sentry.io/platforms/python/configuration/sampling/ . + sample_rate=SENTRY_SAMPLE_RATE, + # + # If you wish to associate users to errors (assuming you are using + # django.contrib.auth) you may enable sending PII data. + send_default_pii=True, + # + # Filter some of the exception which we do not want to see on Sentry. Read more on https://docs.sentry.io/platforms/python/configuration/filtering/ . + before_send=before_send, + # + # Sentry environment should have been configured like this, but I didn't make it work. + # Therefore the Sentry environment is defined as `SENTRY_ENVIRONMENT` in `docker-compose.yml`. + # environment=ENVIRONMENT, + ) # Django allauth configurations @@ -353,6 +388,9 @@ APPLY_DELTAS_LIMIT = 1000 +# the value of the "source" key in each logger entry +LOGGER_SOURCE = os.environ.get("LOGGER_SOURCE", None) + DEBUG_TOOLBAR_CONFIG = { "SHOW_TOOLBAR_CALLBACK": lambda r: DEBUG and ENVIRONMENT == "development", } @@ -393,3 +431,80 @@ ), "Subscription": ("TRIAL_PERIOD_DAYS",), } + + +# `django-auditlog` configurations, read more on https://django-auditlog.readthedocs.io/en/latest/usage.html +AUDITLOG_INCLUDE_TRACKING_MODELS = [ + # NOTE `Delta` and `Job` models are not being automatically audited, because their data changes very often and timestamps are available in their models. + { + "model": "account.emailaddress", + }, + # NOTE Constance model cannot be audited. If enabled, an `IndexError list index out of range` is raised. + # { + # "model": "constance.config", + # }, + { + "model": "core.geodb", + }, + { + "model": "core.organization", + }, + # TODO check if we can use `Organization.members` m2m when next version is released as described in "Many-to-many fields" here https://django-auditlog.readthedocs.io/en/latest/usage.html#automatically-logging-changes + { + "model": "core.organizationmember", + }, + { + "model": "core.person", + "exclude_fields": ["last_login", "updated_at"], + }, + { + "model": "core.project", + # these fields are updated by scripts and will produce a lot of audit noise + "exclude_fields": [ + "updated_at", + "data_last_updated_at", + "data_last_packaged_at", + "last_package_job", + "file_storage_bytes", + ], + }, + # TODO check if we can use `Project.collaborators` m2m when next version is released as described in "Many-to-many fields" here https://django-auditlog.readthedocs.io/en/latest/usage.html#automatically-logging-changes + { + "model": "core.projectcollaborator", + }, + { + "model": "core.secret", + "mask_fields": [ + "value", + ], + }, + # TODO check if we can use `Team.members` m2m when next version is released as described in "Many-to-many fields" here https://django-auditlog.readthedocs.io/en/latest/usage.html#automatically-logging-changes + { + "model": "core.team", + }, + { + "model": "core.teammember", + }, + { + "model": "core.user", + "exclude_fields": ["last_login", "updated_at"], + }, + { + "model": "core.useraccount", + }, + { + "model": "invitations.invitation", + }, + { + "model": "subscription.package", + }, + { + "model": "subscription.packagetype", + }, + { + "model": "subscription.plan", + }, + { + "model": "subscription.subscription", + }, +] diff --git a/docker-app/qfieldcloud/subscription/models.py b/docker-app/qfieldcloud/subscription/models.py index f90439439..c0c1326fd 100644 --- a/docker-app/qfieldcloud/subscription/models.py +++ b/docker-app/qfieldcloud/subscription/models.py @@ -5,6 +5,7 @@ from typing import Optional, Tuple, TypedDict from constance import config +from deprecated import deprecated from django.apps import apps from django.conf import settings from django.core.exceptions import ValidationError @@ -14,7 +15,7 @@ from django.utils import timezone from django.utils.translation import gettext as _ from model_utils.managers import InheritanceManagerMixin -from qfieldcloud.core.models import Person, User, UserAccount +from qfieldcloud.core.models import Organization, Person, User, UserAccount from .exceptions import NotPremiumPlanException @@ -178,6 +179,10 @@ def get_or_create_default(cls) -> "Plan": # updated at updated_at = models.DateTimeField(auto_now=True) + @property + def storage_bytes(self) -> int: + return self.storage_mb * 1000 * 1000 + def save(self, *args, **kwargs): if self.user_type not in (User.Type.PERSON, User.Type.ORGANIZATION): raise ValidationError( @@ -393,9 +398,14 @@ def is_active(self) -> bool: ] @property + @deprecated("Use `AbstractSubscription.active_storage_total_bytes` instead") def active_storage_total_mb(self) -> int: return self.plan.storage_mb + self.active_storage_package_mb + @property + def active_storage_total_bytes(self) -> int: + return self.plan.storage_bytes + self.active_storage_package_bytes + @property def active_storage_package(self) -> Package: return self.get_active_package(PackageType.get_storage_package_type()) @@ -405,12 +415,24 @@ def active_storage_package_quantity(self) -> int: return self.get_active_package_quantity(PackageType.get_storage_package_type()) @property + @deprecated("Use `AbstractSubscription.active_storage_package_bytes` instead") def active_storage_package_mb(self) -> int: return ( self.get_active_package_quantity(PackageType.get_storage_package_type()) * PackageType.get_storage_package_type().unit_amount ) + @property + def active_storage_package_bytes(self) -> int: + return ( + ( + self.get_active_package_quantity(PackageType.get_storage_package_type()) + * PackageType.get_storage_package_type().unit_amount + ) + * 1000 + * 1000 + ) + @property def future_storage_total_mb(self) -> int: return self.plan.storage_mb + self.future_storage_package_mb @@ -655,7 +677,8 @@ def create_default_plan_subscription( ) if account.user.is_organization: - created_by = account.user.organization_owner + # NOTE sometimes `account.user` is not an organization instance for unknown reasons + created_by = Organization.objects.get(pk=account.pk).organization_owner else: created_by = account.user diff --git a/docker-app/qfieldcloud/subscription/tests/test_package.py b/docker-app/qfieldcloud/subscription/tests/test_package.py index 8f7254eba..acc753e44 100644 --- a/docker-app/qfieldcloud/subscription/tests/test_package.py +++ b/docker-app/qfieldcloud/subscription/tests/test_package.py @@ -1,6 +1,5 @@ import logging from datetime import timedelta -from unittest import skip import django.db.utils from django.utils import timezone @@ -658,9 +657,6 @@ def test_used_storage_changes_when_uploading_and_deleting_files_and_versions(sel storage_free_mb=1, ) - @skip( - "Temporary disable the checks for storage limit, as we temporarily disabled purging old versions." - ) def test_api_enforces_storage_limit(self): p1 = Project.objects.create(name="p1", owner=self.u1) @@ -680,9 +676,6 @@ def test_api_enforces_storage_limit(self): ) self.assertEqual(response.status_code, 402) - @skip( - "Temporary disable the checks for storage limit, as we temporarily disabled purging old versions." - ) def test_api_enforces_storage_limit_when_owner_changes(self): plan_10mb = Plan.objects.create( code="plan_10mb", storage_mb=10, is_premium=True diff --git a/docker-app/requirements.txt b/docker-app/requirements.txt index 3829cfdb5..c6aa63032 100644 --- a/docker-app/requirements.txt +++ b/docker-app/requirements.txt @@ -13,11 +13,11 @@ coreschema==0.0.4 cryptography==36.0.1 defusedxml==0.7.1 Deprecated==1.2.13 -Django==3.2.16 +Django==3.2.17 django-allauth==0.44.0 django-appconf==1.0.5 -django-auditlog==1.0a1 -django-axes==5.28.0 +django-auditlog==2.2.2 +django-axes==5.40.1 django-bootstrap4==3.0.1 django-classy-tags==3.0.1 django-common-helpers==0.9.2 @@ -26,13 +26,13 @@ django-countries==7.3.2 django-cron==0.5.0 django-cryptography==1.1 django-currentuser==0.5.3 -django-debug-toolbar==3.4.0 +django-debug-toolbar==3.8.1 django-filter==21.1 -django-invitations==1.9.3 +django-invitations==2.0.0 django-ipware==4.0.2 django-jsonfield==1.4.1 django-migrate-sql-deux==0.4.0 -django-model-utils==4.2.0 +django-model-utils==4.3.1 django-notifications-hq==1.6.0 django-phonenumber-field==7.0.0 django-picklefield==3.1 diff --git a/docker-compose.yml b/docker-compose.yml index 1b1fc1a49..54e833c5a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,8 +31,10 @@ services: SECRET_KEY: ${SECRET_KEY} DEBUG: ${DEBUG} ENVIRONMENT: ${ENVIRONMENT} - # actually I never made it work with sentry_sdk.init(environment=ENVIRONMENT) + SENTRY_DSN: ${SENTRY_DSN} + # Sentry environment should not be configured like this, but I never made it work with `sentry_sdk.init(environment=ENVIRONMENT)`. SENTRY_ENVIRONMENT: ${ENVIRONMENT} + SENTRY_SAMPLE_RATE: ${SENTRY_SAMPLE_RATE} SQL_DATABASE: ${POSTGRES_DB} SQL_DATABASE_TEST: test_${POSTGRES_DB} SQL_USER: ${POSTGRES_USER} @@ -46,7 +48,6 @@ services: STORAGE_REGION_NAME: ${STORAGE_REGION_NAME} STORAGE_ENDPOINT_URL: ${STORAGE_ENDPOINT_URL} QFIELDCLOUD_DEFAULT_NETWORK: ${QFIELDCLOUD_DEFAULT_NETWORK} - SENTRY_DSN: ${SENTRY_DSN} REDIS_PASSWORD: ${REDIS_PASSWORD} GEODB_HOST: ${GEODB_HOST} GEODB_PORT: ${GEODB_PORT} diff --git a/docker-qgis/Dockerfile b/docker-qgis/Dockerfile index 4f1207a08..d0a9b4c3b 100644 --- a/docker-qgis/Dockerfile +++ b/docker-qgis/Dockerfile @@ -1,4 +1,4 @@ -FROM qgis/qgis:final-3_28_2 +FROM qgis/qgis:final-3_28_3 RUN apt-get update \ && apt-get upgrade -y \ diff --git a/scripts/debug.sql b/scripts/debug.sql index a19aa904a..36d591fa1 100644 --- a/scripts/debug.sql +++ b/scripts/debug.sql @@ -21,10 +21,10 @@ CREATE OR REPLACE TEMPORARY VIEW debug_users_vw AS U.is_active, U.last_login, U.date_joined, - U.user_type, - U.has_accepted_tos, - U.has_newsletter_subscription, - UAT.code AS project_owner_plan, + U.type, + -- U.has_accepted_tos, + -- U.has_newsletter_subscription, + -- UAT.code AS project_owner_plan, COALESCE(P.projects_count, 0) AS projects_count, COALESCE(O.organizations_count, 0) AS organizations_count, @@ -33,7 +33,7 @@ CREATE OR REPLACE TEMPORARY VIEW debug_users_vw AS FROM core_user U JOIN core_useraccount UA ON UA.user_id = U.id - JOIN subscription_plan UAT ON UAT.id = UA.plan_id + -- JOIN subscription_plan UAT ON UAT.id = UA.plan_id LEFT JOIN ( SELECT user_id, @@ -96,10 +96,10 @@ CREATE OR REPLACE TEMPORARY VIEW debug_users_slim_vw AS is_active, last_login, date_joined, - user_type, - has_accepted_tos, - has_newsletter_subscription, - project_owner_plan + type + -- has_accepted_tos, + -- has_newsletter_subscription, + -- project_owner_plan FROM debug_users_vw ; @@ -111,8 +111,8 @@ CREATE OR REPLACE TEMPORARY VIEW debug_projects_vw AS P.name AS project_name, LOWER(U.username) AS project_owner_username, U.id AS project_owner_id, - UAT.code AS project_owner_plan, - P.storage_size_mb, + -- UAT.code AS project_owner_plan, + P.file_storage_bytes, P.overwrite_conflicts, P.is_public, P.project_filename, @@ -133,7 +133,7 @@ CREATE OR REPLACE TEMPORARY VIEW debug_projects_vw AS core_project P JOIN core_user U ON U.id = P.owner_id JOIN core_useraccount UA ON UA.user_id = U.id - JOIN subscription_plan UAT ON UAT.id = UA.plan_id + -- JOIN subscription_plan UAT ON UAT.id = UA.plan_id LEFT JOIN ( SELECT project_id, @@ -165,8 +165,8 @@ CREATE OR REPLACE TEMPORARY VIEW debug_projects_slim_vw AS project_name, project_owner_username, project_owner_id, - project_owner_plan, - storage_size_mb, + -- project_owner_plan, + file_storage_bytes, overwrite_conflicts, is_public, project_filename, @@ -188,7 +188,7 @@ CREATE OR REPLACE TEMPORARY VIEW debug_deltas_vw AS P.name AS project_name, LOWER(U.username) AS project_owner_username, U.id AS project_owner_id, - UAT.code AS project_owner_plan, + -- UAT.code AS project_owner_plan, D.id AS delta_id, deltafile_id AS deltafile_id, D.last_status, @@ -210,7 +210,7 @@ CREATE OR REPLACE TEMPORARY VIEW debug_deltas_vw AS JOIN core_project P ON P.id = D.project_id JOIN core_user U ON U.id = P.owner_id JOIN core_useraccount UA ON UA.user_id = U.id - JOIN subscription_plan UAT ON UAT.id = UA.plan_id + -- JOIN subscription_plan UAT ON UAT.id = UA.plan_id JOIN core_user U1 ON U1.id = D.created_by_id LEFT JOIN core_user U2 ON U2.id = D.last_apply_attempt_by_id LEFT JOIN ( @@ -247,7 +247,7 @@ CREATE OR REPLACE TEMPORARY VIEW debug_deltas_slim_vw AS project_name, project_owner_username, project_owner_id, - project_owner_plan, + -- project_owner_plan, delta_id, deltafile_id, last_status, @@ -268,7 +268,7 @@ CREATE OR REPLACE TEMPORARY VIEW debug_jobs_vw AS P.name AS project_name, LOWER(U.username) AS project_owner_username, U.id AS project_owner_id, - UAT.code AS project_owner_plan, + -- UAT.code AS project_owner_plan, J.id AS job_id, J.type, J.status, @@ -290,7 +290,7 @@ CREATE OR REPLACE TEMPORARY VIEW debug_jobs_vw AS JOIN core_project P ON P.id = J.project_id JOIN core_user U ON U.id = P.owner_id JOIN core_useraccount UA ON UA.user_id = U.id - JOIN subscription_plan UAT ON UAT.id = UA.plan_id + -- JOIN subscription_plan UAT ON UAT.id = UA.plan_id JOIN core_user U1 ON U1.id = J.created_by_id LEFT JOIN core_applyjob AJ ON AJ.job_ptr_id = J.id LEFT JOIN ( @@ -311,7 +311,7 @@ CREATE OR REPLACE TEMPORARY VIEW debug_jobs_slim_vw AS project_name, project_owner_username, project_owner_id, - project_owner_plan, + -- project_owner_plan, job_id, type, status,