Merge pull request #434 from Xpirix/handle_various_sentry_errors

Handle various sentry errors

dockerize/Makefile

@@ -236,7 +236,7 @@ devweb-runserver: devweb
 	@echo "------------------------------------------------------------------"
 	@echo "Running in DEVELOPMENT mode"
 	@echo "------------------------------------------------------------------"
-	@docker compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8080
+	@docker compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8081
 
 dbseed:
 	@echo

dockerize/docker-compose.yml

@@ -84,7 +84,7 @@ services:
       - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:rw
     ports:
       # for django test server
-      - "62202:8080"
+      - "62202:8081"
       # for ssh
       - "62203:22"
     networks:
@@ -98,10 +98,18 @@ services:
     restart: unless-stopped
     networks:
       internal:
+    healthcheck:
+      test: ["CMD", "rabbitmqctl", "status"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   beat:
     <<: *uwsgi-common
     container_name: qgis-plugins-beat
+    depends_on:
+      rabbitmq:
+        condition: service_healthy
     working_dir: /home/web/django_project
     entrypoint: [ ]
     command: celery --app=plugins.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO
@@ -112,9 +120,10 @@ services:
     <<: *uwsgi-common
     container_name: qgis-plugins-worker
     depends_on:
-      - db
-      - rabbitmq
-      - beat
+      db:
+        condition: service_started
+      beat:
+        condition: service_started
     working_dir: /home/web/django_project
     entrypoint: []
     command: celery -A plugins worker -l INFO

qgis-app/api/serializers.py

@@ -6,18 +6,17 @@
 from styles.models import Style
 from layerdefinitions.models import LayerDefinition
 from wavefronts.models import Wavefront
-
+from sorl.thumbnail import get_thumbnail
+from django.conf import settings
+from os.path import exists
+from django.templatetags.static import static
 
 class ResourceBaseSerializer(serializers.ModelSerializer):
     creator = serializers.ReadOnlyField(source="get_creator_name")
     resource_type = serializers.SerializerMethodField()
     resource_subtype = serializers.SerializerMethodField()
     thumbnail_full = serializers.ImageField(source="thumbnail_image")
-
-    # A thumbnail image, sorl options and read-only
-    thumbnail = HyperlinkedSorlImageField(
-        "128x128", options={"crop": "center"}, source="thumbnail_image", read_only=True
-    )
+    thumbnail = serializers.SerializerMethodField()
 
     class Meta:
         fields = [
@@ -44,6 +43,23 @@ def get_resource_type(self, obj):
             return "3DModel"
         return self.Meta.model.__name__
 
+    def get_thumbnail(self, obj):
+        request = self.context.get('request')
+        try:
+            if obj.thumbnail_image and exists(obj.thumbnail_image.path):
+                thumbnail = get_thumbnail(obj.thumbnail_image, "128x128", crop="center")
+                if request is not None:
+                    return request.build_absolute_uri(thumbnail.url)
+                return thumbnail.url
+        except Exception as e:
+            pass
+
+        # Return a full URL to a default image if no thumbnail exists or if there's an error
+        default_url = static("images/qgis-icon-32x32.png")
+        if request is not None:
+            return request.build_absolute_uri(default_url)
+        return default_url
+
 
 class GeopackageSerializer(ResourceBaseSerializer):
     class Meta(ResourceBaseSerializer.Meta):

qgis-app/api/tests/test_views.py

@@ -1,6 +1,7 @@
 import io
 import json
 import zipfile
+import os
 
 from django.contrib.auth.models import Group, User
 from django.core.files.base import ContentFile
@@ -275,3 +276,48 @@ def test_download_resource_should_be_a_file_in_a_zip(self):
             self.assertIn("a_filename", zip_file.namelist()[0])
             self.assertNotIn(".zip", zip_file.namelist())
             zip_file.close()
+
+    def test_thumbnail_exists(self):
+        url = reverse("resource-list")
+        response = self.client.get(url)
+        json_parse = json.loads(response.content)
+        self.assertEqual(json_parse["total"], 5)
+        result = json_parse["results"]
+        for i, d in enumerate(result):
+            if d["resource_type"] == "Geopackage":
+                g_index = i
+            elif d["resource_type"] == "Model":
+                m_index = i
+            elif d["resource_type"] == "Style":
+                s_index = i
+            elif d["resource_type"] == "LayerDefinition":
+                l_index = i
+            elif d["resource_type"] == "3DModel":
+                w_index = i
+
+        expected_url = 'http://testserver/media/cache'
+        self.assertTrue(str(result[s_index]['thumbnail']).startswith(expected_url))
+
+    def test_thumbnail_missing(self):
+        # Ensure that the object's thumbnail_image is missing
+        os.remove(self.style.thumbnail_image.path)
+        url = reverse("resource-list")
+        response = self.client.get(url)
+        json_parse = json.loads(response.content)
+        self.assertEqual(json_parse["total"], 5)
+        result = json_parse["results"]
+        for i, d in enumerate(result):
+            if d["resource_type"] == "Geopackage":
+                g_index = i
+            elif d["resource_type"] == "Model":
+                m_index = i
+            elif d["resource_type"] == "Style":
+                s_index = i
+            elif d["resource_type"] == "LayerDefinition":
+                l_index = i
+            elif d["resource_type"] == "3DModel":
+                w_index = i
+
+        expected_url = 'http://testserver/static/images/qgis-icon-32x32.png'
+        print(result[s_index]['thumbnail'])
+        self.assertTrue(str(result[s_index]['thumbnail']).startswith(expected_url))

qgis-app/api/views.py

@@ -105,28 +105,28 @@ class ResourceAPIList(FlatMultipleModelAPIView):
 
     querylist = [
         {
-            "queryset": Geopackage.approved_objects.all(),
-            "serializer_class": GeopackageSerializer,
+            "queryset": LayerDefinition.approved_objects.all(),
+            "serializer_class": LayerDefinitionSerializer,
             "filter_fn": filter_general,
         },
         {
-            "queryset": Model.approved_objects.all(),
-            "serializer_class": ModelSerializer,
+            "queryset": Wavefront.approved_objects.all(),
+            "serializer_class": WavefrontSerializer,
             "filter_fn": filter_general,
         },
         {
-            "queryset": Style.approved_objects.all(),
-            "serializer_class": StyleSerializer,
+            "queryset": Geopackage.approved_objects.all(),
+            "serializer_class": GeopackageSerializer,
             "filter_fn": filter_general,
         },
         {
-            "queryset": LayerDefinition.approved_objects.all(),
-            "serializer_class": LayerDefinitionSerializer,
+            "queryset": Model.approved_objects.all(),
+            "serializer_class": ModelSerializer,
             "filter_fn": filter_general,
         },
         {
-            "queryset": Wavefront.approved_objects.all(),
-            "serializer_class": WavefrontSerializer,
+            "queryset": Style.approved_objects.all(),
+            "serializer_class": StyleSerializer,
             "filter_fn": filter_general,
         },
     ]

qgis-app/base/tests/test_middleware.py

@@ -0,0 +1,31 @@
+from django.test import TestCase, RequestFactory
+from django.template import TemplateDoesNotExist
+from middleware import HandleTemplateDoesNotExistMiddleware
+from django.urls import path
+from django.urls import reverse
+
+class HandleTemplateDoesNotExistMiddlewareTest(TestCase):
+    fixtures = ["fixtures/simplemenu.json"]
+    def setUp(self):
+        # Mock get_response function
+        self.factory = RequestFactory()
+        self.get_response = lambda request: None
+        self.middleware = HandleTemplateDoesNotExistMiddleware(self.get_response)
+
+    def test_template_does_not_exist(self):
+        url = '/planet/template'
+        response = self.client.get(url)
+
+        self.assertEqual(response.status_code, 404)
+        self.assertTemplateUsed(
+            response, '404.html'
+        )
+
+    def test_no_template_error(self):
+        request = self.factory.get('/planet/template')
+
+        # Simulate a different exception
+        response = self.middleware.process_exception(request, Exception("Some other error"))
+
+        # Check that the middleware does not handle this
+        self.assertIsNone(response)

qgis-app/base/views/processing_view.py

@@ -172,6 +172,11 @@ class ResourceSearchMixin(object):
     """
 
     def get_queryset_search(self, qs):
+        # Allowed fields for ordering
+        allowed_order_by_fields = [
+            "name", "type", "download_count"
+            "creator", "upload_date", "modified_date"
+        ]
         q = self.request.GET.get("q")
         if q:
             qs = qs.annotate(
@@ -192,7 +197,8 @@ def get_queryset_search(self, qs):
             elif order_by == "type":
                 qs = qs.order_by("style_type__name")
             else:
-                qs = qs.order_by(order_by)
+                if order_by.lstrip('-') in allowed_order_by_fields:
+                    qs = qs.order_by(order_by)
         return qs
 
     def get_queryset_search_and_is_creator(self, qs):

qgis-app/middleware.py

@@ -1,4 +1,14 @@
 # -*- coding:utf-8 -*-
+# myapp/middleware.py
+
+from django.template import TemplateDoesNotExist
+from django.shortcuts import render
+from django.core.exceptions import RequestDataTooBig
+from django.http import JsonResponse
+from django.utils.deprecation import MiddlewareMixin
+import logging
+import sentry_sdk
+
 """
     QGIS-DJANGO - MIDDLEWARE
 
@@ -7,6 +17,7 @@
     @license: GNU AGPL, see COPYING for details.
 """
 
+logger = logging.getLogger(__name__)
 
 def XForwardedForMiddleware(get_response):
     # One-time configuration and initialization.
@@ -28,3 +39,33 @@ def middleware(request):
         return response
 
     return middleware
+
+class HandleTemplateDoesNotExistMiddleware(MiddlewareMixin):
+    """Handle missing templates"""
+    def process_exception(self, request, exception):
+        if isinstance(exception, TemplateDoesNotExist):
+            return render(request, '404.html', status=404)
+        return None
+
+class HandleOSErrorMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        try:
+            response = self.get_response(request)
+        except OSError as e:
+            logger.error("OSError occurred", exc_info=True)
+            sentry_sdk.capture_exception(e)
+            raise e
+        return response
+class HandleRequestDataTooBigMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        try:
+            response = self.get_response(request)
+            return response
+        except RequestDataTooBig:
+            return JsonResponse({'error': 'Request data is too large. Please upload smaller files.'}, status=413)

qgis-app/plugins/tasks/__init__.py

@@ -1,3 +1,4 @@
 from plugins.tasks.generate_plugins_xml import *  # noqa
 from plugins.tasks.update_feedjack import *  # noqa
 from plugins.tasks.update_qgis_versions import *  # noqa
+from plugins.tasks.rebuild_search_index import *  # noqa

qgis-app/plugins/tasks/rebuild_search_index.py

@@ -0,0 +1,10 @@
+from celery import shared_task
+from celery.utils.log import get_task_logger
+
+logger = get_task_logger(__name__)
+
+
+@shared_task
+def rebuild_index():
+    import subprocess
+    subprocess.call(['python', 'manage.py', 'rebuild_index'])

qgis-app/plugins/templatetags/plugin_utils.py

@@ -1,5 +1,6 @@
 from django import template
 from PIL import Image, UnidentifiedImageError
+import xml.etree.ElementTree as ET
 
 register = template.Library()
 
@@ -34,8 +35,23 @@ def file_extension(value):
 def is_image_valid(image):
     if not image:
         return False
+    # Check if the file is an SVG by extension
+    if image.path.lower().endswith('.svg'):
+        return _validate_svg(image.path)
+    return _validate_image(image.path)
+
+
+def _validate_svg(file_path):
+    try:
+        # Parse the SVG file to ensure it's well-formed XML
+        ET.parse(file_path)
+        return True
+    except (ET.ParseError, FileNotFoundError):
+        return False
+
+def _validate_image(file_path):
     try:
-        img = Image.open(image.path)
+        img = Image.open(file_path)
         img.verify()
         return True
     except (FileNotFoundError, UnidentifiedImageError):

qgis-app/plugins/tests/test_download.py

@@ -65,4 +65,21 @@ def test_version_download_per_country(self):
 
         self.assertEqual(response.status_code, 200)
         self.assertTrue(download_record.country_code == 'ID')
-        self.assertTrue(download_record.country_name == 'Indonesia')
+        self.assertTrue(download_record.country_name == 'Indonesia')
+
+
+    def test_download_per_country_with_invalid_ip(self):
+        download_url = reverse('version_download', args=[self.plugin.package_name, self.version.version])
+        c = Client(REMOTE_ADDR='123.456.789.100')
+        response = c.get(download_url)
+
+        self.version.refresh_from_db()
+        self.plugin.refresh_from_db()
+        download_record = PluginVersionDownload.objects.get(
+            plugin_version=self.version, 
+            download_date=timezone.now().date()
+        )
+
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(download_record.country_code == 'N/D')
+        self.assertTrue(download_record.country_name == 'N/D')

qgis-app/plugins/views.py

@@ -1534,6 +1534,10 @@ def version_download(request, package_name, version):
             country_code = 'N/D'
             country_name = 'N/D'
 
+    # Handle null values
+    country_code = country_code or 'N/D'
+    country_name = country_name or 'N/D'
+
     download_record, created = PluginVersionDownload.objects.get_or_create(
         plugin_version = version, 
         country_code = country_code,