From c7282fa1c4cbf95536f560b29316af36280c96cb Mon Sep 17 00:00:00 2001 From: Stanislav Khlud Date: Tue, 14 Jan 2025 12:40:37 +0700 Subject: [PATCH] Set up export/import action mixin --- docs/api_drf.rst | 18 ++ docs/getting_started.rst | 14 ++ import_export_extensions/api/__init__.py | 10 +- .../api/mixins/__init__.py | 3 + .../api/{mixins.py => mixins/common.py} | 5 +- .../api/mixins/export_mixins.py | 141 ++++++++++++ .../api/mixins/import_mixins.py | 115 ++++++++++ .../api/serializers/export_job.py | 12 +- .../api/serializers/import_job.py | 12 +- .../api/views/__init__.py | 4 + .../api/views/export_job.py | 198 +++++------------ .../api/views/import_job.py | 200 +++++++----------- test_project/fake_app/api/views.py | 46 +++- .../integration_tests/test_api/test_export.py | 146 ++++++++++++- .../integration_tests/test_api/test_import.py | 165 +++++++++++++-- test_project/urls.py | 17 ++ 16 files changed, 809 insertions(+), 297 deletions(-) create mode 100644 import_export_extensions/api/mixins/__init__.py rename import_export_extensions/api/{mixins.py => mixins/common.py} (76%) create mode 100644 import_export_extensions/api/mixins/export_mixins.py create mode 100644 import_export_extensions/api/mixins/import_mixins.py diff --git a/docs/api_drf.rst b/docs/api_drf.rst index 00b8bdb..be65d48 100644 --- a/docs/api_drf.rst +++ b/docs/api_drf.rst @@ -14,9 +14,27 @@ API (Rest Framework) .. autoclass:: import_export_extensions.api.ExportJobForUserViewSet :members: +.. autoclass:: import_export_extensions.api.BaseImportJobViewSet + :members: + +.. autoclass:: import_export_extensions.api.BaseExportJobViewSet + :members: + +.. autoclass:: import_export_extensions.api.BaseImportJobForUserViewSet + :members: + +.. autoclass:: import_export_extensions.api.BaseExportJobForUserViewSet + :members: + .. autoclass:: import_export_extensions.api.LimitQuerySetToCurrentUserMixin :members: +.. autoclass:: import_export_extensions.api.ImportStartActionMixin + :members: + +.. autoclass:: import_export_extensions.api.ExportStartActionMixin + :members: + .. autoclass:: import_export_extensions.api.CreateExportJob :members: create, validate diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 38a1931..2f8a696 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -169,3 +169,17 @@ the OpenAPI specification will be available. .. figure:: _static/images/bands-openapi.png A screenshot of the generated OpenAPI specification + + +Import/Export API actions mixins +----------------- + +Alternatively you can use ``api.mixins.ExportStartActionMixin`` and ``api.mixins.ImportStartActionMixin`` +to add to your current viewsets ability to create import/export jobs. +You would also need to use ``api.views.BaseExportJobViewSet/BaseExportJobForUsersViewSet`` +and ``api.views.BaseExportJobViewSet/BaseImportJobForUsersViewSet`` to setup endpoints to be able to: + +* ``list`` - Returns a list of jobs for the ``resource_class`` set in ViewSet +* ``retrieve`` - Returns details of a job based on the provided ID +* ``cancel`` - Stops the import/export process and sets the job's status to ``CANCELLED``. +* ``confirm`` - Confirms the import after the parse stage. This action is available only in import jobs. diff --git a/import_export_extensions/api/__init__.py b/import_export_extensions/api/__init__.py index 392cb62..b5f6e73 100644 --- a/import_export_extensions/api/__init__.py +++ b/import_export_extensions/api/__init__.py @@ -1,4 +1,8 @@ -from .mixins import LimitQuerySetToCurrentUserMixin +from .mixins import ( + ExportStartActionMixin, + ImportStartActionMixin, + LimitQuerySetToCurrentUserMixin, +) from .serializers import ( CreateExportJob, CreateImportJob, @@ -8,6 +12,10 @@ ProgressSerializer, ) from .views import ( + BaseExportJobForUserViewSet, + BaseExportJobViewSet, + BaseImportJobForUserViewSet, + BaseImportJobViewSet, ExportJobForUserViewSet, ExportJobViewSet, ImportJobForUserViewSet, diff --git a/import_export_extensions/api/mixins/__init__.py b/import_export_extensions/api/mixins/__init__.py new file mode 100644 index 0000000..dbbac86 --- /dev/null +++ b/import_export_extensions/api/mixins/__init__.py @@ -0,0 +1,3 @@ +from .common import LimitQuerySetToCurrentUserMixin +from .export_mixins import ExportStartActionMixin +from .import_mixins import ImportStartActionMixin diff --git a/import_export_extensions/api/mixins.py b/import_export_extensions/api/mixins/common.py similarity index 76% rename from import_export_extensions/api/mixins.py rename to import_export_extensions/api/mixins/common.py index c0952a1..63008f6 100644 --- a/import_export_extensions/api/mixins.py +++ b/import_export_extensions/api/mixins/common.py @@ -3,7 +3,10 @@ class LimitQuerySetToCurrentUserMixin: def get_queryset(self): """Return user's jobs.""" - if self.action == "start": + if self.action in ( + getattr(self, "import_action", ""), + getattr(self, "export_action", ""), + ): # To make it consistent and for better support of drf-spectacular return super().get_queryset() # pragma: no cover return ( diff --git a/import_export_extensions/api/mixins/export_mixins.py b/import_export_extensions/api/mixins/export_mixins.py new file mode 100644 index 0000000..a3af1da --- /dev/null +++ b/import_export_extensions/api/mixins/export_mixins.py @@ -0,0 +1,141 @@ +import collections.abc +import contextlib +import typing + +from django.conf import settings +from django.utils import module_loading + +from rest_framework import ( + decorators, + request, + response, + status, +) + +from ... import resources +from .. import serializers + + +class ExportStartActionMixin: + """Mixin which adds start export action.""" + + resource_class: type[resources.CeleryModelResource] + export_action = "start_export_action" + export_action_name = "export" + export_action_url = "export" + export_detail_serializer_class = serializers.ExportJobSerializer + export_ordering: collections.abc.Sequence[str] = () + export_ordering_fields: collections.abc.Sequence[str] = () + export_open_api_description = ( + "This endpoint creates export job and starts it. " + "To monitor progress use detail endpoint for jobs to fetch state of " + "job. Once it's status is `EXPORTED`, you can download file." + ) + + def __init_subclass__(cls) -> None: + super().__init_subclass__() + # Skip if it is has no resource_class specified + if not hasattr(cls, "resource_class"): + return + filter_backends = [ + module_loading.import_string( + settings.DRF_EXPORT_DJANGO_FILTERS_BACKEND, + ), + ] + if cls.export_ordering_fields: + filter_backends.append( + module_loading.import_string( + settings.DRF_EXPORT_ORDERING_BACKEND, + ), + ) + + def start_export_action( + self: "ExportStartActionMixin", + request: request.Request, + ) -> response.Response: + return self.start_export(request) + + setattr(cls, cls.export_action, start_export_action) + decorators.action( + methods=["POST"], + url_name=cls.export_action_name, + url_path=cls.export_action_url, + detail=False, + queryset=cls.resource_class.get_model_queryset(), + serializer_class=cls().get_export_create_serializer_class(), + filterset_class=getattr( + cls.resource_class, + "filterset_class", + None, + ), + filter_backends=filter_backends, + ordering=cls.export_ordering, + ordering_fields=cls.export_ordering_fields, + )(getattr(cls, cls.export_action)) + # Correct specs of drf-spectacular if it is installed + with contextlib.suppress(ImportError): + from drf_spectacular import utils + + utils.extend_schema_view( + **{ + cls.export_action: utils.extend_schema( + description=cls.export_open_api_description, + filters=True, + responses={ + status.HTTP_201_CREATED: cls().get_export_detail_serializer_class(), # noqa: E501 + }, + ), + }, + )(cls) + + def get_queryset(self): + """Return export model queryset on export action. + + For better openapi support and consistency. + + """ + if self.action == self.export_action: + return self.resource_class.get_model_queryset() # pragma: no cover + return super().get_queryset() + + def get_export_detail_serializer_class(self): + """Get serializer which will be used show details of export job.""" + return self.export_detail_serializer_class + + def get_export_create_serializer_class(self): + """Get serializer which will be used to start export job.""" + return serializers.get_create_export_job_serializer( + self.resource_class, + ) + + def get_export_resource_kwargs(self) -> dict[str, typing.Any]: + """Provide extra arguments to resource class.""" + return {} + + def get_serializer(self, *args, **kwargs): + """Provide resource kwargs to serializer class.""" + if self.action == self.export_action: + kwargs.setdefault( + "resource_kwargs", + self.get_export_resource_kwargs(), + ) + return super().get_serializer(*args, **kwargs) + + def start_export(self, request: request.Request) -> response.Response: + """Validate request data and start ExportJob.""" + ordering = request.query_params.get("ordering", "") + if ordering: + ordering = ordering.split(",") + serializer = self.get_serializer( + data=request.data, + ordering=ordering, + filter_kwargs=request.query_params, + ) + serializer.is_valid(raise_exception=True) + export_job = serializer.save() + return response.Response( + data=self.get_export_detail_serializer_class()( + instance=export_job, + ).data, + status=status.HTTP_201_CREATED, + ) diff --git a/import_export_extensions/api/mixins/import_mixins.py b/import_export_extensions/api/mixins/import_mixins.py new file mode 100644 index 0000000..3bce3f4 --- /dev/null +++ b/import_export_extensions/api/mixins/import_mixins.py @@ -0,0 +1,115 @@ +import contextlib +import typing + +from rest_framework import ( + decorators, + request, + response, + status, +) + +from ... import resources +from .. import serializers + + +class ImportStartActionMixin: + """Mixin which adds start import action.""" + + resource_class: type[resources.CeleryModelResource] + import_action = "start_import_action" + import_action_name = "import" + import_action_url = "import" + import_detail_serializer_class = serializers.ImportJobSerializer + import_open_api_description = ( + "This endpoint creates import job and starts it. " + "To monitor progress use detail endpoint for jobs to fetch state of " + "job. Once it's status is `PARSED`, you can confirm import and data " + "should start importing. When status `INPUT_ERROR` or `PARSE_ERROR` " + "it means data failed validations and can't be imported. " + "When status is `IMPORTED`, it means data is in system and " + "job is completed." + ) + + def __init_subclass__(cls) -> None: + super().__init_subclass__() + # Skip if it is has no resource_class specified + if not hasattr(cls, "resource_class"): + return + + def start_import_action( + self: "ImportStartActionMixin", + request: request.Request, + ) -> response.Response: + return self.start_import(request) + + setattr(cls, cls.import_action, start_import_action) + decorators.action( + methods=["POST"], + url_name=cls.import_action_name, + url_path=cls.import_action_url, + detail=False, + queryset=cls.resource_class.get_model_queryset(), + serializer_class=cls().get_import_create_serializer_class(), + )(getattr(cls, cls.import_action)) + # Correct specs of drf-spectacular if it is installed + with contextlib.suppress(ImportError): + from drf_spectacular import utils + + utils.extend_schema_view( + **{ + cls.import_action: utils.extend_schema( + description=cls.import_open_api_description, + filters=True, + responses={ + status.HTTP_201_CREATED: cls().get_import_detail_serializer_class(), # noqa: E501 + }, + ), + }, + )(cls) + + def get_queryset(self): + """Return import model queryset on import action. + + For better openapi support and consistency. + + """ + if self.action == self.import_action: + return self.resource_class.get_model_queryset() # pragma: no cover + return super().get_queryset() + + def get_import_detail_serializer_class(self): + """Get serializer which will be used show details of import job.""" + return self.import_detail_serializer_class + + def get_import_create_serializer_class(self): + """Get serializer which will be used to start import job.""" + return serializers.get_create_import_job_serializer( + self.resource_class, + ) + + def get_import_resource_kwargs(self) -> dict[str, typing.Any]: + """Provide extra arguments to resource class.""" + return {} + + def get_serializer(self, *args, **kwargs): + """Provide resource kwargs to serializer class.""" + if self.action == self.import_action: + kwargs.setdefault( + "resource_kwargs", + self.get_import_resource_kwargs(), + ) + return super().get_serializer(*args, **kwargs) + + def start_import(self, request: request.Request) -> response.Response: + """Validate request data and start ImportJob.""" + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + import_job = serializer.save() + + return response.Response( + data=self.get_import_detail_serializer_class()( + instance=import_job, + ).data, + status=status.HTTP_201_CREATED, + ) diff --git a/import_export_extensions/api/serializers/export_job.py b/import_export_extensions/api/serializers/export_job.py index 6a8cc6c..e2b38b9 100644 --- a/import_export_extensions/api/serializers/export_job.py +++ b/import_export_extensions/api/serializers/export_job.py @@ -99,10 +99,19 @@ def update(self, instance, validated_data): """Empty method to pass linters checks.""" +# Use it to cache already generated serializers to avoid duplication +_GENERATED_EXPORT_JOB_SERIALIZERS: dict[ + type[resources.CeleryModelResource], + type, +] = {} + + def get_create_export_job_serializer( resource: type[resources.CeleryModelResource], ) -> type: """Create serializer for ExportJobs creation.""" + if resource in _GENERATED_EXPORT_JOB_SERIALIZERS: + return _GENERATED_EXPORT_JOB_SERIALIZERS[resource] class _CreateExportJob(CreateExportJob): """Serializer to start export job.""" @@ -116,8 +125,9 @@ class _CreateExportJob(CreateExportJob): ], ) - return type( + _GENERATED_EXPORT_JOB_SERIALIZERS[resource] = type( f"{resource.__name__}CreateExportJob", (_CreateExportJob,), {}, ) + return _GENERATED_EXPORT_JOB_SERIALIZERS[resource] diff --git a/import_export_extensions/api/serializers/import_job.py b/import_export_extensions/api/serializers/import_job.py index 5bb02ad..c52b3d8 100644 --- a/import_export_extensions/api/serializers/import_job.py +++ b/import_export_extensions/api/serializers/import_job.py @@ -126,18 +126,28 @@ def update(self, instance, validated_data): """Empty method to pass linters checks.""" +# Use it to cache already generated serializers to avoid duplication +_GENERATED_IMPORT_JOB_SERIALIZERS: dict[ + type[resources.CeleryModelResource], + type, +] = {} + + def get_create_import_job_serializer( resource: type[resources.CeleryModelResource], ) -> type: """Create serializer for ImportJobs creation.""" + if resource in _GENERATED_IMPORT_JOB_SERIALIZERS: + return _GENERATED_IMPORT_JOB_SERIALIZERS[resource] class _CreateImportJob(CreateImportJob): """Serializer to start import job.""" resource_class: type[resources.CeleryModelResource] = resource - return type( + _GENERATED_IMPORT_JOB_SERIALIZERS[resource] = type( f"{resource.__name__}CreateImportJob", (_CreateImportJob,), {}, ) + return _GENERATED_IMPORT_JOB_SERIALIZERS[resource] diff --git a/import_export_extensions/api/views/__init__.py b/import_export_extensions/api/views/__init__.py index 2409e38..2ddfa4e 100644 --- a/import_export_extensions/api/views/__init__.py +++ b/import_export_extensions/api/views/__init__.py @@ -1,8 +1,12 @@ from .export_job import ( + BaseExportJobForUserViewSet, + BaseExportJobViewSet, ExportJobForUserViewSet, ExportJobViewSet, ) from .import_job import ( + BaseImportJobForUserViewSet, + BaseImportJobViewSet, ImportJobForUserViewSet, ImportJobViewSet, ) diff --git a/import_export_extensions/api/views/export_job.py b/import_export_extensions/api/views/export_job.py index 16b3061..9e1aafa 100644 --- a/import_export_extensions/api/views/export_job.py +++ b/import_export_extensions/api/views/export_job.py @@ -1,9 +1,5 @@ import collections.abc import contextlib -import typing - -from django.conf import settings -from django.utils import module_loading from rest_framework import ( decorators, @@ -14,87 +10,80 @@ status, viewsets, ) -from rest_framework.request import Request import django_filters -from ... import models, resources +from ... import models from .. import mixins as core_mixins -from .. import serializers -class ExportBase(type): - """Add custom create action for each ExportJobViewSet.""" +class BaseExportJobViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + """Base viewset for managing export jobs.""" - def __new__(cls, name, bases, attrs, **kwargs): - """Dynamically create an export start api endpoint. + permission_classes = (permissions.IsAuthenticated,) + serializer_class = core_mixins.ExportStartActionMixin.export_detail_serializer_class # noqa: E501 + queryset = models.ExportJob.objects.all() + filterset_class: django_filters.rest_framework.FilterSet | None = None + search_fields: collections.abc.Sequence[str] = ("id",) + ordering: collections.abc.Sequence[str] = ( + "id", + ) + ordering_fields: collections.abc.Sequence[str] = ( + "id", + "created", + "modified", + ) - We need this to specify on fly action's filterset_class and queryset - (django-filters requires view's queryset and filterset_class's - queryset model to match). Also, if drf-spectacular is installed - specify request and response, and enable filters. + def __init_subclass__(cls) -> None: + """Dynamically create an cancel api endpoints. + + Need to do this to enable action and correct open-api spec generated by + drf_spectacular. """ - viewset: type[ExportJobViewSet] = super().__new__( - cls, - name, - bases, - attrs, - **kwargs, - ) - # Skip if it is has no resource_class specified - if not hasattr(viewset, "resource_class"): - return viewset - filter_backends = [ - module_loading.import_string(settings.DRF_EXPORT_DJANGO_FILTERS_BACKEND), - ] - if viewset.export_ordering_fields: - filter_backends.append( - module_loading.import_string(settings.DRF_EXPORT_ORDERING_BACKEND), - ) - decorators.action( - methods=["POST"], - detail=False, - queryset=viewset.resource_class.get_model_queryset(), - filterset_class=getattr( - viewset.resource_class, "filterset_class", None, - ), - filter_backends=filter_backends, - ordering=viewset.export_ordering, - ordering_fields=viewset.export_ordering_fields, - )(viewset.start) + super().__init_subclass__() decorators.action( methods=["POST"], detail=True, - )(viewset.cancel) + )(cls.cancel) # Correct specs of drf-spectacular if it is installed with contextlib.suppress(ImportError): from drf_spectacular.utils import extend_schema, extend_schema_view - - detail_serializer_class = viewset().get_detail_serializer_class() - return extend_schema_view( - start=extend_schema( - filters=True, - request=viewset().get_export_create_serializer_class(), - responses={ - status.HTTP_201_CREATED: detail_serializer_class, - }, - ), + if hasattr(cls, "get_export_detail_serializer_class"): + response_serializer = cls().get_export_detail_serializer_class() # noqa: E501 + else: + response_serializer = cls().get_serializer_class() + extend_schema_view( cancel=extend_schema( request=None, responses={ - status.HTTP_200_OK: detail_serializer_class, + status.HTTP_200_OK: response_serializer, }, ), - )(viewset) - return viewset + )(cls) + def cancel(self, *args, **kwargs) -> response.Response: + """Cancel export job that is in progress.""" + job: models.ExportJob = self.get_object() + + try: + job.cancel_export() + except ValueError as error: + raise exceptions.ValidationError(error.args[0]) from error + + serializer = self.get_serializer(instance=job) + return response.Response( + status=status.HTTP_200_OK, + data=serializer.data, + ) class ExportJobViewSet( - mixins.ListModelMixin, - mixins.RetrieveModelMixin, - viewsets.GenericViewSet, - metaclass=ExportBase, + core_mixins.ExportStartActionMixin, + BaseExportJobViewSet, ): """Base API viewset for ExportJob model. @@ -106,95 +95,24 @@ class ExportJobViewSet( """ - permission_classes = (permissions.IsAuthenticated,) - queryset = models.ExportJob.objects.all() - serializer_class = serializers.ExportJobSerializer - resource_class: type[resources.CeleryModelResource] - filterset_class: django_filters.rest_framework.FilterSet | None = None - search_fields: collections.abc.Sequence[str] = ("id",) - ordering: collections.abc.Sequence[str] = ( - "id", - ) - ordering_fields: collections.abc.Sequence[str] = ( - "id", - "created", - "modified", - ) - export_ordering: collections.abc.Sequence[str] = () - export_ordering_fields: collections.abc.Sequence[str] = () + export_action_name = "start" + export_action_url = "start" def get_queryset(self): """Filter export jobs by resource used in viewset.""" - if self.action == "start": - # To make it consistent and for better support of drf-spectacular - return super().get_queryset() # pragma: no cover return super().get_queryset().filter( resource_path=self.resource_class.class_path, ) - def get_resource_kwargs(self) -> dict[str, typing.Any]: - """Provide extra arguments to resource class.""" - return {} - - def get_serializer(self, *args, **kwargs): - """Provide resource kwargs to serializer class.""" - if self.action == "start": - kwargs.setdefault("resource_kwargs", self.get_resource_kwargs()) - return super().get_serializer(*args, **kwargs) - - def get_serializer_class(self): - """Return special serializer on creation.""" - if self.action == "start": - return self.get_export_create_serializer_class() - return self.get_detail_serializer_class() - - def get_detail_serializer_class(self): - """Get serializer which will be used show details of export job.""" - return self.serializer_class - - def get_export_create_serializer_class(self): - """Get serializer which will be used to start export job.""" - return serializers.get_create_export_job_serializer( - self.resource_class, - ) - - def start(self, request: Request): - """Validate request data and start ExportJob.""" - ordering = request.query_params.get("ordering", "") - if ordering: - ordering = ordering.split(",") - serializer = self.get_serializer( - data=request.data, - ordering=ordering, - filter_kwargs=request.query_params, - ) - serializer.is_valid(raise_exception=True) - export_job = serializer.save() - return response.Response( - data=self.get_detail_serializer_class()( - instance=export_job, - ).data, - status=status.HTTP_201_CREATED, - ) - - def cancel(self, *args, **kwargs): - """Cancel export job that is in progress.""" - job: models.ExportJob = self.get_object() - - try: - job.cancel_export() - except ValueError as error: - raise exceptions.ValidationError(error.args[0]) from error - - serializer = self.get_serializer(instance=job) - return response.Response( - status=status.HTTP_200_OK, - data=serializer.data, - ) - class ExportJobForUserViewSet( core_mixins.LimitQuerySetToCurrentUserMixin, ExportJobViewSet, ): """Viewset for providing export feature to users.""" + +class BaseExportJobForUserViewSet( + core_mixins.LimitQuerySetToCurrentUserMixin, + BaseExportJobViewSet, +): + """Viewset for providing export job management to users.""" diff --git a/import_export_extensions/api/views/import_job.py b/import_export_extensions/api/views/import_job.py index be7bcfb..174dfe3 100644 --- a/import_export_extensions/api/views/import_job.py +++ b/import_export_extensions/api/views/import_job.py @@ -1,5 +1,5 @@ +import collections import contextlib -import typing from rest_framework import ( decorators, @@ -11,153 +11,67 @@ viewsets, ) -from ... import models, resources +from ... import models from .. import mixins as core_mixins -from .. import serializers -class ImportBase(type): - """Add custom create action for each ImportJobViewSet.""" +class BaseImportJobViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + """Base viewset for managing import jobs.""" - def __new__(cls, name, bases, attrs, **kwargs): - """Dynamically create an import start api endpoint. + permission_classes = (permissions.IsAuthenticated,) + serializer_class = core_mixins.ImportStartActionMixin.import_detail_serializer_class # noqa: E501 + queryset = models.ImportJob.objects.all() + search_fields: collections.abc.Sequence[str] = ("id",) + ordering: collections.abc.Sequence[str] = ( + "id", + ) + ordering_fields: collections.abc.Sequence[str] = ( + "id", + "created", + "modified", + ) - If drf-spectacular is installed - specify request and response, and enable filters. + def __init_subclass__(cls) -> None: + """Dynamically create an cancel api endpoints. - """ - viewset: type[ImportJobViewSet] = super().__new__( - cls, - name, - bases, - attrs, - **kwargs, - ) - # Skip if it is has no resource_class specified - if not hasattr(viewset, "resource_class"): - return viewset + Need to do this to enable action and correct open-api spec generated by + drf_spectacular. - decorators.action( - methods=["POST"], - detail=False, - )(viewset.start) + """ + super().__init_subclass__() decorators.action( methods=["POST"], detail=True, - )(viewset.confirm) + )(cls.cancel) decorators.action( methods=["POST"], detail=True, - )(viewset.cancel) - + )(cls.confirm) # Correct specs of drf-spectacular if it is installed with contextlib.suppress(ImportError): from drf_spectacular.utils import extend_schema, extend_schema_view - - detail_serializer_class = viewset().get_detail_serializer_class() - return extend_schema_view( - start=extend_schema( - request=viewset().get_import_create_serializer_class(), - responses={ - status.HTTP_201_CREATED: detail_serializer_class, - }, - ), - confirm=extend_schema( + if hasattr(cls, "get_import_detail_serializer_class"): + response_serializer = cls().get_import_detail_serializer_class() # noqa: E501 + else: + response_serializer = cls().get_serializer_class() + extend_schema_view( + cancel=extend_schema( request=None, responses={ - status.HTTP_200_OK: detail_serializer_class, + status.HTTP_200_OK: response_serializer, }, ), - cancel=extend_schema( + confirm=extend_schema( request=None, responses={ - status.HTTP_200_OK: detail_serializer_class, + status.HTTP_200_OK: response_serializer, }, ), - )(viewset) - return viewset - - -class ImportJobViewSet( - mixins.ListModelMixin, - mixins.RetrieveModelMixin, - viewsets.GenericViewSet, - metaclass=ImportBase, -): - """Base API viewset for ImportJob model. - - Based on resource_class it will generate an endpoint which will allow to - start an import to model which was specified in resource_class. On success - this endpoint we return an instance of import. - - Endpoints: - list - to get list of all import jobs - details(retrieve) - to get status of import job - start - create import job and start parsing data from attached file - confirm - confirm import after parsing process is finished - cancel - stop importing/parsing process and cancel this import job - - """ - - permission_classes = (permissions.IsAuthenticated,) - queryset = models.ImportJob.objects.all() - serializer_class = serializers.ImportJobSerializer - resource_class: type[resources.CeleryModelResource] - search_fields = ("id",) - ordering = ( - "id", - ) - ordering_fields = ( - "id", - "created", - "modified", - ) - - def get_queryset(self): - """Filter import jobs by resource used in viewset.""" - return super().get_queryset().filter( - resource_path=self.resource_class.class_path, - ) - - def get_resource_kwargs(self) -> dict[str, typing.Any]: - """Provide extra arguments to resource class.""" - return {} - - def get_serializer(self, *args, **kwargs): - """Provide resource kwargs to serializer class.""" - if self.action == "start": - kwargs.setdefault("resource_kwargs", self.get_resource_kwargs()) - return super().get_serializer(*args, **kwargs) - - def get_serializer_class(self): - """Return special serializer on creation.""" - if self.action == "start": - return self.get_import_create_serializer_class() - return self.get_detail_serializer_class() - - def get_detail_serializer_class(self): - """Get serializer which will be used show details of import job.""" - return self.serializer_class - - def get_import_create_serializer_class(self): - """Get serializer which will be used to start import job.""" - return serializers.get_create_import_job_serializer( - self.resource_class, - ) - - def start(self, request, *args, **kwargs): - """Validate request data and start ImportJob.""" - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - - import_job = serializer.save() - - return response.Response( - data=self.get_detail_serializer_class()( - instance=import_job, - ).data, - status=status.HTTP_201_CREATED, - ) + )(cls) def confirm(self, *args, **kwargs): """Confirm import job that has `parsed` status.""" @@ -189,8 +103,46 @@ def cancel(self, *args, **kwargs): data=serializer.data, ) +class ImportJobViewSet( + core_mixins.ImportStartActionMixin, + BaseImportJobViewSet, +): + """Base API viewset for ImportJob model. + + Based on resource_class it will generate an endpoint which will allow to + start an import to model which was specified in resource_class. On success + this endpoint we return an instance of import. + + Endpoints: + list - to get list of all import jobs + details(retrieve) - to get status of import job + start - create import job and start parsing data from attached file + confirm - confirm import after parsing process is finished + cancel - stop importing/parsing process and cancel this import job + + """ + + import_action_name = "start" + import_action_url = "start" + + def get_queryset(self): + """Filter import jobs by resource used in viewset.""" + return super().get_queryset().filter( + resource_path=self.resource_class.class_path, + ) + + def start(self, request, *args, **kwargs): + """Validate request data and start ImportJob.""" + return self._start_import(request) + class ImportJobForUserViewSet( core_mixins.LimitQuerySetToCurrentUserMixin, ImportJobViewSet, ): """Viewset for providing import feature to users.""" + +class BaseImportJobForUserViewSet( + core_mixins.LimitQuerySetToCurrentUserMixin, + BaseImportJobViewSet, +): + """Viewset for providing export job management to users.""" diff --git a/test_project/fake_app/api/views.py b/test_project/fake_app/api/views.py index a5e860e..b88cc98 100644 --- a/test_project/fake_app/api/views.py +++ b/test_project/fake_app/api/views.py @@ -1,18 +1,52 @@ -from import_export_extensions.api import views +from rest_framework import mixins, serializers, viewsets -from ..resources import SimpleArtistResource +from import_export_extensions import api +from .. import models, resources -class ArtistExportViewSet(views.ExportJobForUserViewSet): + +class ArtistExportViewSet(api.ExportJobForUserViewSet): """Simple ViewSet for exporting Artist model.""" - resource_class = SimpleArtistResource + resource_class = resources.SimpleArtistResource export_ordering_fields = ( "id", "name", ) -class ArtistImportViewSet(views.ImportJobForUserViewSet): +class ArtistImportViewSet(api.ImportJobForUserViewSet): """Simple ViewSet for importing Artist model.""" - resource_class = SimpleArtistResource + resource_class = resources.SimpleArtistResource + +class ArtistSerializer(serializers.ModelSerializer): + """Serializer for Artist model.""" + + class Meta: + model = models.Artist + fields = ( + "id", + "name", + "instrument", + ) + +class ArtistViewSet( + api.ExportStartActionMixin, + api.ImportStartActionMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + """Simple viewset for Artist model.""" + + resource_class = resources.SimpleArtistResource + queryset = models.Artist.objects.all() + serializer_class = ArtistSerializer + filterset_class = resources.SimpleArtistResource.filterset_class + ordering = ( + "id", + ) + ordering_fields = ( + "id", + "name", + ) diff --git a/test_project/tests/integration_tests/test_api/test_export.py b/test_project/tests/integration_tests/test_api/test_export.py index 1cd12be..d3ad63a 100644 --- a/test_project/tests/integration_tests/test_api/test_export.py +++ b/test_project/tests/integration_tests/test_api/test_export.py @@ -11,12 +11,26 @@ @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="export_url", + argvalues=[ + pytest.param( + reverse("export-artist-start"), + id="Export url", + ), + pytest.param( + f"{reverse('artists-export')}", + id="Action url", + ), + ], +) def test_export_api_creates_export_job( admin_api_client: test.APIClient, + export_url: str, ): """Ensure export start API creates new export job.""" response = admin_api_client.post( - path=reverse("export-artist-start"), + path=export_url, data={ "file_format": "csv", }, @@ -53,15 +67,29 @@ def test_export_api_creates_export_job( ), ], ) +@pytest.mark.parametrize( + argnames="export_url", + argvalues=[ + pytest.param( + reverse("export-artist-start"), + id="Export url", + ), + pytest.param( + f"{reverse('artists-export')}", + id="Action url", + ), + ], +) def test_export_api_filtering( admin_api_client: test.APIClient, filter_query: str, filter_name: str, filter_value: str, + export_url: str, ): """Ensure export start API passes filter kwargs correctly.""" response = admin_api_client.post( - path=f"{reverse('export-artist-start')}?{filter_query}", + path=f"{export_url}?{filter_query}", data={ "file_format": "csv", }, @@ -100,14 +128,28 @@ def test_export_api_filtering( ), ], ) +@pytest.mark.parametrize( + argnames="export_url", + argvalues=[ + pytest.param( + reverse("export-artist-start"), + id="Export url", + ), + pytest.param( + f"{reverse('artists-export')}", + id="Action url", + ), + ], +) def test_export_api_ordering( admin_api_client: test.APIClient, ordering_query: str, ordering_value: collections.abc.Sequence[str], + export_url: str, ): """Ensure export start API passes ordering correctly.""" response = admin_api_client.post( - path=f"{reverse('export-artist-start')}?{ordering_query}", + path=f"{export_url}?{ordering_query}", data={ "file_format": "csv", }, @@ -123,12 +165,26 @@ def test_export_api_ordering( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="export_url", + argvalues=[ + pytest.param( + f"{reverse('export-artist-start')}?id=invalid_id", + id="Export url with invalid filter_kwargs", + ), + pytest.param( + f"{reverse('artists-export')}?id=invalid_id", + id="Action url with invalid filter_kwargs", + ), + ], +) def test_export_api_create_export_job_with_invalid_filter_kwargs( admin_api_client: test.APIClient, + export_url: str, ): """Ensure export start API with invalid kwargs return an error.""" response = admin_api_client.post( - path=f"{reverse('export-artist-start')}?id=invalid_id", + path=export_url, data={ "file_format": "csv", }, @@ -138,12 +194,26 @@ def test_export_api_create_export_job_with_invalid_filter_kwargs( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="export_url", + argvalues=[ + pytest.param( + f"{reverse('export-artist-start')}?ordering=invalid_id", + id="Export url with invalid ordering", + ), + pytest.param( + f"{reverse('artists-export')}?ordering=invalid_id", + id="Action url with invalid ordering", + ), + ], +) def test_export_api_create_export_job_with_invalid_ordering( admin_api_client: test.APIClient, + export_url: str, ): """Ensure export start API with invalid ordering return an error.""" response = admin_api_client.post( - path=f"{reverse('export-artist-start')}?ordering=invalid_id", + path=export_url, data={ "file_format": "csv", }, @@ -156,14 +226,28 @@ def test_export_api_create_export_job_with_invalid_ordering( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="export_url", + argvalues=[ + pytest.param( + "export-artist-detail", + id="Model url", + ), + pytest.param( + "export-jobs-detail", + id="General url", + ), + ], +) def test_export_api_detail( admin_api_client: test.APIClient, artist_export_job: ExportJob, + export_url: str, ): """Ensure export detail API shows current export job status.""" response = admin_api_client.get( path=reverse( - "export-artist-detail", + export_url, kwargs={"pk": artist_export_job.id}, ), ) @@ -172,15 +256,29 @@ def test_export_api_detail( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="export_url", + argvalues=[ + pytest.param( + "export-artist-detail", + id="Model url", + ), + pytest.param( + "export-jobs-detail", + id="General url", + ), + ], +) def test_import_user_api_get_detail( user: User, admin_api_client: test.APIClient, artist_export_job: ExportJob, + export_url: str, ): """Ensure import detail api for user returns only users jobs.""" response = admin_api_client.get( path=reverse( - "export-artist-detail", + export_url, kwargs={"pk": artist_export_job.id}, ), ) @@ -190,7 +288,7 @@ def test_import_user_api_get_detail( artist_export_job.save() response = admin_api_client.get( path=reverse( - "export-artist-detail", + export_url, kwargs={"pk": artist_export_job.id}, ), ) @@ -205,17 +303,31 @@ def test_import_user_api_get_detail( ExportJob.ExportStatus.EXPORTING, ], ) +@pytest.mark.parametrize( + argnames="export_url", + argvalues=[ + pytest.param( + "export-artist-cancel", + id="Model url", + ), + pytest.param( + "export-jobs-cancel", + id="General url", + ), + ], +) def test_export_api_cancel( admin_api_client: test.APIClient, artist_export_job: ExportJob, allowed_cancel_status: ExportJob.ExportStatus, + export_url: str, ): """Ensure that export canceled with allowed statuses.""" artist_export_job.export_status = allowed_cancel_status artist_export_job.save() response = admin_api_client.post( path=reverse( - "export-artist-cancel", + export_url, kwargs={"pk": artist_export_job.pk}, ), ) @@ -233,17 +345,31 @@ def test_export_api_cancel( ExportJob.ExportStatus.CANCELLED, ], ) +@pytest.mark.parametrize( + argnames="export_url", + argvalues=[ + pytest.param( + "export-artist-cancel", + id="Model url", + ), + pytest.param( + "export-jobs-cancel", + id="General url", + ), + ], +) def test_export_api_cancel_with_errors( admin_api_client: test.APIClient, artist_export_job: ExportJob, incorrect_job_status: ExportJob.ExportStatus, + export_url: str, ): """Ensure that export job with incorrect statuses cannot be canceled.""" artist_export_job.export_status = incorrect_job_status artist_export_job.save() response = admin_api_client.post( path=reverse( - "export-artist-cancel", + export_url, kwargs={"pk": artist_export_job.pk}, ), ) diff --git a/test_project/tests/integration_tests/test_api/test_import.py b/test_project/tests/integration_tests/test_api/test_import.py index 2bd8bc3..c4b3011 100644 --- a/test_project/tests/integration_tests/test_api/test_import.py +++ b/test_project/tests/integration_tests/test_api/test_import.py @@ -14,14 +14,28 @@ @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="import_url", + argvalues=[ + pytest.param( + reverse("import-artist-start"), + id="Model url", + ), + pytest.param( + f"{reverse('artists-import')}", + id="Action url", + ), + ], +) def test_import_api_creates_import_job( admin_api_client: APIClient, uploaded_file: SimpleUploadedFile, + import_url: str, ): """Ensure import start api creates new import job.""" import_job_count = ImportJob.objects.count() response = admin_api_client.post( - path=reverse("import-artist-start"), + path=import_url, data={ "file": uploaded_file, }, @@ -33,14 +47,28 @@ def test_import_api_creates_import_job( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="import_url", + argvalues=[ + pytest.param( + "import-artist-detail", + id="Model url", + ), + pytest.param( + "import-jobs-detail", + id="General url", + ), + ], +) def test_import_api_detail( admin_api_client: APIClient, artist_import_job: ImportJob, + import_url: str, ): """Ensure import detail api shows current import job status.""" response = admin_api_client.get( path=reverse( - "import-artist-detail", + import_url, kwargs={"pk": artist_import_job.id}, ), ) @@ -52,7 +80,7 @@ def test_import_api_detail( response = admin_api_client.get( path=reverse( - "import-artist-detail", + import_url, kwargs={"pk": artist_import_job.id}, ), ) @@ -61,15 +89,29 @@ def test_import_api_detail( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="import_url", + argvalues=[ + pytest.param( + "import-artist-detail", + id="Model url", + ), + pytest.param( + "import-jobs-detail", + id="General url", + ), + ], +) def test_import_user_api_get_detail( user: User, admin_api_client: APIClient, artist_import_job: ImportJob, + import_url: str, ): """Ensure import detail api for user returns only users jobs.""" response = admin_api_client.get( path=reverse( - "import-artist-detail", + import_url, kwargs={"pk": artist_import_job.id}, ), ) @@ -87,10 +129,23 @@ def test_import_user_api_get_detail( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="import_url", + argvalues=[ + pytest.param( + "import-artist-detail", + id="Model url", + ), + pytest.param( + "import-jobs-detail", + id="General url", + ), + ], +) def test_force_import_api_detail( admin_api_client: APIClient, - superuser: User, force_import_artist_job: ImportJob, + import_url: str, ): """Test detail api for force import job. @@ -101,7 +156,7 @@ def test_force_import_api_detail( """ response = admin_api_client.get( path=reverse( - "import-artist-detail", + import_url, kwargs={"pk": force_import_artist_job.id}, ), ) @@ -113,7 +168,7 @@ def test_force_import_api_detail( response = admin_api_client.get( path=reverse( - "import-artist-detail", + import_url, kwargs={"pk": force_import_artist_job.id}, ), ) @@ -126,10 +181,24 @@ def test_force_import_api_detail( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="import_url", + argvalues=[ + pytest.param( + "import-artist-detail", + id="Model url", + ), + pytest.param( + "import-jobs-detail", + id="General url", + ), + ], +) def test_import_api_detail_with_row_errors( admin_api_client: APIClient, existing_artist: Artist, superuser: User, + import_url: str, ): """Ensure import detail api shows row errors.""" expected_error_message = "Instrument matching query does not exist." @@ -150,7 +219,7 @@ def test_import_api_detail_with_row_errors( response = admin_api_client.get( path=reverse( - "import-artist-detail", + import_url, kwargs={"pk": import_artist_job.pk}, ), ) @@ -164,10 +233,24 @@ def test_import_api_detail_with_row_errors( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="import_url", + argvalues=[ + pytest.param( + "import-artist-detail", + id="Model url", + ), + pytest.param( + "import-jobs-detail", + id="General url", + ), + ], +) def test_import_api_detail_with_base_errors( superuser: User, admin_api_client: APIClient, existing_artist: Artist, + import_url: str, ): """Ensure import detail api shows base errors.""" expected_error_message = ( @@ -193,7 +276,7 @@ def test_import_api_detail_with_base_errors( response = admin_api_client.get( path=reverse( - "import-artist-detail", + import_url, kwargs={"pk": import_artist_job.pk}, ), ) @@ -207,16 +290,30 @@ def test_import_api_detail_with_base_errors( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames="import_url", + argvalues=[ + pytest.param( + "import-artist-confirm", + id="Model url", + ), + pytest.param( + "import-jobs-confirm", + id="General url", + ), + ], +) def test_import_api_confirm_parsed_job( admin_api_client: APIClient, artist_import_job: ImportJob, + import_url: str, ): """Check that parsed import job can be confirmed.""" artist_import_job.parse_data() artist_import_job.refresh_from_db() response = admin_api_client.post( path=reverse( - "import-artist-confirm", + import_url, kwargs={"pk": artist_import_job.pk}, ), ) @@ -240,10 +337,24 @@ def test_import_api_confirm_parsed_job( ImportJob.ImportStatus.CANCELLED, ], ) +@pytest.mark.parametrize( + argnames="import_url", + argvalues=[ + pytest.param( + "import-artist-confirm", + id="Model url", + ), + pytest.param( + "import-jobs-confirm", + id="General url", + ), + ], +) def test_import_api_confirm_incorrect_job_status( admin_api_client: APIClient, artist_import_job: ImportJob, incorrect_job_status: ImportJob.ImportStatus, + import_url: str, ): """Ensure that not parsed job can't be confirmed.""" artist_import_job.import_status = incorrect_job_status @@ -251,7 +362,7 @@ def test_import_api_confirm_incorrect_job_status( response = admin_api_client.post( path=reverse( - "import-artist-confirm", + import_url, kwargs={"pk": artist_import_job.pk}, ), ) @@ -273,17 +384,31 @@ def test_import_api_confirm_incorrect_job_status( ImportJob.ImportStatus.CONFIRMED, ], ) +@pytest.mark.parametrize( + argnames="import_url", + argvalues=[ + pytest.param( + "import-artist-cancel", + id="Model url", + ), + pytest.param( + "import-jobs-cancel", + id="General url", + ), + ], +) def test_import_api_cancel_job( admin_api_client: APIClient, artist_import_job: ImportJob, allowed_cancel_status: ImportJob.ImportStatus, + import_url: str, ): """Check that import job with allowed statuses can be cancelled.""" artist_import_job.import_status = allowed_cancel_status artist_import_job.save() response = admin_api_client.post( path=reverse( - "import-artist-cancel", + import_url, kwargs={"pk": artist_import_job.pk}, ), ) @@ -303,10 +428,24 @@ def test_import_api_cancel_job( ImportJob.ImportStatus.CANCELLED, ], ) +@pytest.mark.parametrize( + argnames="import_url", + argvalues=[ + pytest.param( + "import-artist-cancel", + id="Model url", + ), + pytest.param( + "import-jobs-cancel", + id="General url", + ), + ], +) def test_import_api_cancel_incorrect_job_status( admin_api_client: APIClient, artist_import_job: ImportJob, incorrect_job_status: ImportJob.ImportStatus, + import_url: str, ): """Ensure that import job with incorrect statuses cannot be canceled.""" artist_import_job.import_status = incorrect_job_status @@ -314,7 +453,7 @@ def test_import_api_cancel_incorrect_job_status( response = admin_api_client.post( path=reverse( - "import-artist-cancel", + import_url, kwargs={"pk": artist_import_job.pk}, ), ) diff --git a/test_project/urls.py b/test_project/urls.py index 413e23c..5818293 100644 --- a/test_project/urls.py +++ b/test_project/urls.py @@ -7,6 +7,8 @@ from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView +from import_export_extensions import api + from .fake_app.api import views ie_router = DefaultRouter() @@ -15,11 +17,26 @@ views.ArtistExportViewSet, basename="export-artist", ) +ie_router.register( + "export-jobs", + api.BaseExportJobForUserViewSet, + basename="export-jobs", +) +ie_router.register( + "import-jobs", + api.BaseImportJobForUserViewSet, + basename="import-jobs", +) ie_router.register( "import-artist", views.ArtistImportViewSet, basename="import-artist", ) +ie_router.register( + "artists", + views.ArtistViewSet, + basename="artists", +) urlpatterns = [re_path("^admin/", admin.site.urls), *ie_router.urls]