feat: Move user export task on celery

docker-compose.local.yml

@@ -47,6 +47,10 @@ services:
       - /app/__pypackages__
     env_file:
       - ./packages/backend/.env
+    environment:
+      - AWS_ACCESS_KEY_ID=foo
+      - AWS_SECRET_ACCESS_KEY=bar
+      - AWS_DEFAULT_REGION=eu-west-1
 
   workers:
     volumes:

packages/backend/.test.env

@@ -26,6 +26,7 @@ STRIPE_LIVE_MODE=False
 DJSTRIPE_WEBHOOK_SECRET=whsec_12345
 
 AWS_STORAGE_BUCKET_NAME=test-bucket
+AWS_EXPORTS_STORAGE_BUCKET_NAME=exports-bucket
 AWS_S3_CUSTOM_DOMAIN=cdn.example.com
 AWS_ENDPOINT_URL=
 

packages/backend/Dockerfile

@@ -21,7 +21,7 @@ RUN apt-get update && apt-get install -y gcc postgresql-client ca-certificates j
   && pip install --no-cache-dir setuptools pdm~=2.5.2 awscli==1.32.24
 
 
-RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+RUN curl -fsS https://deb.nodesource.com/setup_20.x | bash - \
   && apt-get --no-install-recommends install -y nodejs
 
 COPY --from=chamber /chamber /bin/chamber

packages/backend/apps/demo/tests/factories.py

@@ -1,4 +1,5 @@
 import factory
+from django.core.files.base import ContentFile
 
 from .. import models
 
@@ -17,6 +18,9 @@ class Meta:
 
 class DocumentDemoItemFactory(factory.django.DjangoModelFactory):
     created_by = factory.SubFactory(user_factories.UserFactory)
+    file = factory.LazyAttribute(
+        lambda _: ContentFile(factory.django.ImageField()._make_data({'width': 1024, 'height': 768}), name='test.jpg')
+    )
 
     class Meta:
         model = models.DocumentDemoItem

packages/backend/apps/users/admin.py

@@ -87,8 +87,7 @@ def export_user_data(self, request, queryset):
             "user_ids": [str(user_id) for user_id in queryset.values_list("id", flat=True)],
             "admin_email": request.user.email,
         }
-        export_user_data_task = tasks.ExportUserData()
-        export_user_data_task.apply(data=data)
+        tasks.export_user_data.apply_async((data['user_ids'], data['admin_email']))
 
         self.message_user(request, "Exported user data will be sent to you via e-mail", messages.SUCCESS)
 

packages/backend/apps/users/services/export/email_serializers.py

@@ -0,0 +1,14 @@
+from rest_framework import serializers
+
+
+class UserDataExportEmailBaseSerializer(serializers.Serializer):
+    email = serializers.CharField()
+    export_url = serializers.CharField()
+
+
+class UserDataExportEmailSerializer(serializers.Serializer):
+    data = UserDataExportEmailBaseSerializer()
+
+
+class AdminDataExportEmailSerializer(serializers.Serializer):
+    data = UserDataExportEmailBaseSerializer(many=True)

packages/backend/apps/users/services/export/emails.py

@@ -1,11 +1,18 @@
-from common.emails import get_send_email_event
-from userauth.constants import UserEmails
-from userauth.types import ExportedUserData
+from common import emails
+from . import email_serializers
+from . import constants
 
 
-def get_user_export_email_event(to: str, data: ExportedUserData):
-    return get_send_email_event(detail_type=UserEmails.USER_EXPORT.value, data={"to": to, "data": data})
+class DataExportEmail(emails.Email):
+    def __init__(self, to: str, data: dict):
+        super().__init__(to=to, data=data)
 
 
-def get_admin_export_email_event(to: str, data: list[ExportedUserData]):
-    return get_send_email_event(detail_type=UserEmails.USER_EXPORT_ADMIN.value, data={"to": to, "data": data})
+class UserDataExportEmail(DataExportEmail):
+    name = constants.UserEmails.USER_EXPORT
+    serializer_class = email_serializers.UserDataExportEmailSerializer
+
+
+class AdminDataExportEmail(DataExportEmail):
+    name = constants.UserEmails.USER_EXPORT_ADMIN
+    serializer_class = email_serializers.AdminDataExportEmailSerializer

packages/backend/apps/users/services/export/services/export.py

@@ -3,26 +3,35 @@
 import json
 import zipfile
 from typing import Union
+from django.conf import settings
 
 import boto3
 from ..protocols import UserDataExportable, UserFilesExportable
 from ..constants import ExportUserArchiveRootPaths
 from ....models import User
-
+from utils import hashid
 
 
 class CrudDemoItemDataExport(UserDataExportable):
     export_key = "crud_demo_items"
 
     @classmethod
     def export(cls, user: User) -> list[str]:
-        return [cls.schema_class.from_orm(item).json() for item in user.cruddemoitem_set]
+        return [
+            json.dumps(
+                {
+                    "id": hashid.encode(item.id),
+                    "name": item.name,
+                }
+            )
+            for item in user.cruddemoitem_set.all()
+        ]
 
 
 class DocumentDemoItemFileExport(UserFilesExportable):
     @classmethod
     def export(cls, user: User) -> list[str]:
-        return [document.file for document in user.documents]
+        return [document.file.name for document in user.documents.all()]
 
 
 class UserDataExport(UserDataExportable):
@@ -31,7 +40,21 @@ class UserDataExport(UserDataExportable):
 
     @classmethod
     def export(cls, user: User) -> Union[str, list[str]]:
-        return cls.schema_class.from_orm(user).json()
+        user_data = {
+            "id": hashid.encode(user.id),
+            "profile": {
+                "id": user.profile.id,
+                "first_name": user.profile.first_name,
+                "last_name": user.profile.last_name,
+            },
+            "email": user.email,
+            "is_superuser": user.is_superuser,
+            "is_active": user.is_active,
+            "is_confirmed": user.is_confirmed,
+            "created": user.created.isoformat(),
+        }
+
+        return json.dumps(user_data)
 
 
 class ExportUserArchive:
@@ -80,7 +103,7 @@ def _export_user_archive_to_zip(self, user_data: dict, user_files: list[str]) ->
 
             for file_path in user_files:
                 with io.BytesIO() as buffer:
-                    s3.download_fileobj(settings.AWS_STORAGE_BUCKET_NAME, file_path, buffer)
+                    s3.download_fileobj(settings.AWS_STORAGE_BUCKET_NAME, file_path.name, buffer)
                     zf.writestr(f"{self._user_id}/{file_path}", buffer.getvalue())
 
         return archive_filename

packages/backend/apps/users/services/export/services/user.py

@@ -1,9 +1,8 @@
 from typing import Optional
-
-
-import boto3
 from typing import TypedDict
+
 from . import export
+from .. import emails
 from ....models import User
 
 
@@ -15,22 +14,20 @@ class ExportedUserData(TypedDict):
 class _ProcessUserDataExport:
     def __call__(self, user_ids: list[str], admin_email: str):
         entries = []
-        email_events = []
 
         for user_id in user_ids:
             if user := self._get_user(user_id):
                 entry = self._get_user_export_entry(user)
-                email_events.append(emails.get_user_export_email_event(to=user.email, data=entry))
+                emails.UserDataExportEmail(to=user.email, data={"data": entry}).send()
                 entries.append(entry)
 
         if entries:
-            email_events.append(emails.get_admin_export_email_event(to=admin_email, data=entries))
-            self._send_export_emails(email_events)
+            emails.AdminDataExportEmail(to=admin_email, data={"data": entries}).send()
 
     @staticmethod
     def _get_user(user_id: str) -> Optional[User]:
         try:
-            user = User.objects.prefetch_related('profile','cruddemoitem_set','documents').get(id=user_id)
+            user = User.objects.prefetch_related('profile', 'cruddemoitem_set', 'documents').get(id=user_id)
             return user
         except User.DoesNotExist:
             return None
@@ -40,10 +37,5 @@ def _get_user_export_entry(user: User) -> ExportedUserData:
         export_user_archive = export.ExportUserArchive(user)
         return {"email": user.email, "export_url": export_user_archive.run()}
 
-    @staticmethod
-    def _send_export_emails(email_events: list):
-        events = boto3.client('events', endpoint_url=settings.AWS_ENDPOINT_URL)
-        events.put_events(Entries=email_events)
-
 
 process_user_data_export = _ProcessUserDataExport()

packages/backend/apps/users/tasks.py

@@ -1,7 +1,7 @@
 import importlib
 
 from django.conf import settings
-from celery import shared_task, states
+from celery import shared_task
 from .services.export.services import user as user_services
 
 module_name, package = settings.LAMBDA_TASKS_BASE_HANDLER.rsplit(".", maxsplit=1)
@@ -14,5 +14,5 @@ def __init__(self):
 
 
 @shared_task(bind=True)
-def export_user_data(user_ids, admin_email):
+def export_user_data(self, user_ids, admin_email):
     user_services.process_user_data_export(user_ids=user_ids, admin_email=admin_email)

packages/backend/apps/users/tests/test_services.py

@@ -1,7 +1,19 @@
 import pytest
+import datetime
+import io
+import os
+import json
+import zipfile
+import boto3
+from moto import mock_s3
+from unittest.mock import call
+
+from django.conf import settings
 from apps.users.exceptions import OTPVerificationFailure
 from apps.users.models import User
+from utils import hashid
 from ..services.otp import validate_otp
+from ..services.export.services.export import ExportUserArchive
 
 pytestmark = pytest.mark.django_db
 
@@ -21,3 +33,89 @@ def test_invalid_token_raises_exception(self, user_factory, totp_mock):
             validate_otp(user, "token")
 
         assert str(error.value) == "Verification token is invalid"
+
+
+class TestExportUserArchive:
+    @pytest.fixture
+    def user(self, user_factory):
+        return user_factory.create()
+
+    @pytest.fixture
+    def export_user_archive(self, user) -> ExportUserArchive:
+        return ExportUserArchive(user=user)
+
+    @pytest.fixture
+    def mocked_zip_file(self, mocker):
+        zip_file_mock = mocker.MagicMock(spec=zipfile.ZipFile)
+        mocker.patch("zipfile.ZipFile", return_value=zip_file_mock)
+        mocked_zip_file = zip_file_mock.__enter__.return_value = mocker.Mock()
+
+        return mocked_zip_file
+
+    @pytest.fixture
+    def file_cleanup(self, user):
+        hashed_user_id = hashid.encode(user.id)
+        yield
+        os.remove(f"/tmp/{hashed_user_id}.zip")
+
+    def test_user_data_is_exported(self, user, export_user_archive):
+        data = export_user_archive._export_user_data()
+        assert data.get("user") == json.dumps(
+            {
+                "id": hashid.encode(user.id),
+                "profile": {
+                    "id": user.profile.id,
+                    "first_name": user.profile.first_name,
+                    "last_name": user.profile.last_name,
+                },
+                "email": user.email,
+                "is_superuser": user.is_superuser,
+                "is_active": user.is_active,
+                "is_confirmed": user.is_confirmed,
+                "created": user.created.isoformat(),
+            }
+        )
+
+    def test_crud_demo_items_data_is_exported(self, user, crud_demo_item_factory, export_user_archive):
+        item = crud_demo_item_factory.create(created_by=user)
+        data = export_user_archive._export_user_data()
+        assert data.get("crud_demo_items") == [json.dumps({"id": hashid.encode(item.id), "name": item.name})]
+
+    def test_document_demo_item_is_exported(self, user, document_demo_item_factory, export_user_archive):
+        document = document_demo_item_factory.create(created_by=user)
+        data = export_user_archive._export_user_files()
+        assert document.file in data
+
+    @mock_s3
+    def test_zip_archive_is_created(self, user, document_demo_item_factory, mocked_zip_file, export_user_archive):
+        s3 = boto3.client("s3", region_name='us-east-1', endpoint_url=settings.AWS_S3_ENDPOINT_URL)
+        s3.create_bucket(Bucket=settings.AWS_EXPORTS_STORAGE_BUCKET_NAME)
+        document_item = document_demo_item_factory.create()
+        user_data = {"user": "data"}
+        document_content = b"content"
+        with io.BytesIO() as document_file:
+            document_file.write(document_content)
+            document_file.seek(0)
+            s3.upload_fileobj(document_file, settings.AWS_STORAGE_BUCKET_NAME, document_item.file.name)
+        hashed_user_id = hashid.encode(user.id)
+
+        archive_file_path = export_user_archive._export_user_archive_to_zip(
+            user_data=user_data, user_files=[document_item.file]
+        )
+
+        assert archive_file_path == f"/tmp/{hashed_user_id}.zip"
+        assert [
+            call.writestr(f'{hashed_user_id}/{hashed_user_id}.json', json.dumps(user_data).encode('utf-8')),
+            call.writestr(f'{hashed_user_id}/{document_item.file}', document_content),
+        ] in mocked_zip_file.mock_calls
+
+    @pytest.mark.usefixtures('file_cleanup', 's3_exports_bucket')
+    @pytest.mark.freeze_time
+    def test_user_archive_export_url_is_generated(self, user, export_user_archive):
+        timestamp = datetime.datetime.now().strftime("%d-%m-%y_%H-%M-%S")
+        expected_obj_key = f"exports/{hashid.encode(user.id)}_{timestamp}.zip"
+
+        export_url = export_user_archive.run()
+
+        assert settings.AWS_EXPORTS_STORAGE_BUCKET_NAME in export_url
+        assert expected_obj_key in export_url