Update django to 4.2.0, update python to 3.11

.env-github-actions

@@ -7,6 +7,7 @@ AML_SECRET_KEY=amlsecretkey
 AML_LOCATION_PROVIDER=http://ip2country:5000/{}
 AML_DEBUG=True
 DJANGO_SETTINGS_MODULE=aml.development_settings
+CSRF_TRUSTED_ORIGINS=http://localhost:3000
 
 FRONTEND_API_ROOT=http://localhost:8000
 FRONTEND_EXPERIMENT_SLUG=gold-msi

.env.dist

@@ -14,6 +14,7 @@ DJANGO_SUPERUSER_PASSWORD=admin # do not use in production!
 DJANGO_SUPERUSER_EMAIL=mail@example.com # do not use in production!
 AML_LOCATION_PROVIDER=http://ip2country:5000/{} # address of the ip2country container, don't change
 AML_ALLOWED_HOSTS="localhost" # needs to be changed when running in production
+CSRF_TRUSTED_ORIGINS=http://localhost:3000 # needs to be changed when running in production
 
 FRONTEND_API_ROOT=http://localhost:8000 # address of the server, don't change
 FRONTEND_EXPERIMENT_SLUG=gold-msi # experiment slug that the frontend redirects to

backend/.flake8

@@ -4,42 +4,72 @@
 # High Priority: Security and Correctness
 # These issues are critical and should be addressed first.
 extend-ignore =
-    E722, # Do not use bare 'except'
-    F722, # Syntax error identified by pyflakes
-    F821, # Undefined name
+    # Do not use bare 'except'
+    E722,
+    # Syntax error identified by pyflakes
+    F722,
+     # Undefined name 
+    F821,
 
 # Medium Priority: Code Maintainability and Readability
 # Improving these can greatly enhance code readability and maintainability.
-    E501, # Line too long
-    F401, # Unused import
-    F403, # 'from module import *' used; unable to detect undefined names
-    F405, # Name may be undefined, or defined from star imports
-    F811, # Redefinition of unused name from line N
-    F841, # Local variable name is assigned to but never used
+    # Line too long
+    E501,
+    # Unused import
+    F401,
+    # 'from module import *' used; unable to detect undefined names
+    F403,
+     # Name may be undefined, or defined from star imports
+    F405,
+    # Redefinition of unused name from line N
+    F811,
+    # Local variable name is assigned to but never used
+    F841,
 
 # Low Priority: Style Guide Adherence
 # These are mostly about whitespace and indentation, which can be adjusted later.
-    E201, # Whitespace after '('
-    E202, # Whitespace before ')'
-    E203, # Whitespace before ':'
-    E222, # Multiple spaces after operator
-    E225, # Missing whitespace around operator
-    E231, # Missing whitespace after ','
-    E251, # Unexpected spaces around keyword / parameter equals
-    E262, # Inline comment should start with '# '
-    E122, # Continuation line missing indentation or outdented
-    E124, # Closing bracket does not match visual indentation
-    E125, # Continuation line with same indent as next logical line
-    E127, # Continuation line over-indented for visual indent
-    E128, # Continuation line under-indented for visual indent
-    E131, # Continuation line unaligned for hanging indent
-    E261, # At least two spaces before inline comment
-    W191, # Indentation contains tabs
-    W291, # Trailing whitespace
-    W292, # No newline at end of file
-    W293, # Blank line contains whitespace
-    W391, # Blank line at end of file
-    W503, # Line break occurred before a binary operator
+    # Whitespace after '('
+    E201,
+    # Whitespace before ')'
+    E202,
+    # Whitespace before ':'
+    E203,
+    # Multiple spaces after operator
+    E222,
+    # Missing whitespace around operator
+    E225,
+    # Missing whitespace after ','
+    E231,
+    # Unexpected spaces around keyword / parameter equals
+    E251,
+    # Inline comment should start with '# '
+    E262,
+    # Continuation line missing indentation or outdented
+    E122,
+    # Closing bracket does not match visual indentation
+    E124,
+    # Continuation line with same indent as next logical line
+    E125,
+    # Continuation line over-indented for visual indent
+    E127,
+    # Continuation line under-indented for visual indent
+    E128,
+    # Continuation line unaligned for hanging indent
+    E131,
+    # At least two spaces before inline comment
+    E261,
+    # Indentation contains tabs
+    W191,
+    # Trailing whitespace
+    W291,
+    # No newline at end of file
+    W292,
+    # Blank line contains whitespace
+    W293,
+    # Blank line at end of file
+    W391,
+    # Line break occurred before a binary operator
+    W503,
 
 # General Configuration
 max-line-length = 120

backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/python:3.8
+FROM docker.io/python:3.11
 ENV PYTHONUNBUFFERED 1
 RUN apt-get -y update && apt-get install -y ffmpeg gettext
 

backend/DockerfileDevelop

@@ -1,4 +1,4 @@
-FROM docker.io/python:3.8 as base
+FROM docker.io/python:3.11 as base
 ENV PYTHONUNBUFFERED 1
 RUN apt-get -y update && apt-get install -y ffmpeg gettext
 

backend/aml/base_settings.py

@@ -15,6 +15,11 @@
 from corsheaders.defaults import default_headers
 import sentry_sdk
 
+# Workaround for deprecated ugettext_lazy in django-inline-actions
+import django
+from django.utils.translation import gettext_lazy
+django.utils.translation.ugettext_lazy = gettext_lazy
+
 logger = logging.getLogger(__name__)
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -156,6 +161,7 @@
     "http://localhost:3000,http://127.0.0.1:3000,{}".format(HOMEPAGE)
 ).split(",")
 
+
 # CORS
 CORS_ORIGIN_ALLOW_ALL = False
 CORS_ALLOW_CREDENTIALS = True
@@ -165,6 +171,8 @@
     'baggage',
 ]
 
+CSRF_TRUSTED_ORIGINS = [os.getenv('CSRF_TRUSTED_ORIGINS')]
+
 SESSION_SAVE_EVERY_REQUEST = False # Do not set to True, because it will break session-based participant_id
 
 CSRF_USE_SESSIONS = False

backend/experiment/rules/base.py

@@ -164,7 +164,7 @@ def social_media_info(self, experiment, score):
             'hashtags': [experiment.hashtag or experiment.slug, "amsterdammusiclab", "citizenscience"]
         }
 
-    def validate_playlist(self, playlist: Playlist):
+    def validate_playlist(self, playlist: None):
         errors = []
         # Common validations across experiments
         if not playlist:
@@ -177,7 +177,7 @@ def validate_playlist(self, playlist: Playlist):
             errors.append('The experiment must have at least one section.')
 
         try:
-            Playlist.clean_csv(playlist)
+            playlist.clean_csv()
         except ValidationError as e:
             errors += e.error_list
 

backend/experiment/rules/eurovision_2020.py

@@ -34,11 +34,11 @@ def plan_sections(self, session):
         n_free = session.experiment.rounds - 2 * n_old - n_new
 
         # Assign songs.
-        old_songs = random.sample(old_new_song_set, k=n_old)
-        free_songs = random.sample(free_song_set - set(old_songs), k=n_free)
+        old_songs = random.sample(list(old_new_song_set), k=n_old)
+        free_songs = random.sample(list(free_song_set - set(old_songs)), k=n_free)
         new_songs = \
             random.sample(
-                free_song_set - set(old_songs + free_songs),
+                list(free_song_set - set(old_songs + free_songs)),
                 k=n_new
             )
 

backend/experiment/rules/tests/test_base.py

@@ -66,6 +66,6 @@ def test_validate_playlist(self):
         errors = base.validate_playlist(playlist)
         self.assertEqual(errors, ['The experiment must have a playlist.'])
 
-        playlist = Playlist()
+        playlist = Playlist.objects.create()
         errors = base.validate_playlist(playlist)
-        self.assertEqual(errors, ['The experiment must have at least one section.'])
+        self.assertEqual(errors, ['The experiment must have at least one section.'])

backend/experiment/tests/test_admin_experiment.py

@@ -26,9 +26,6 @@ class MockRequest:
 
 request = MockRequest()
 
-this_experiment_admin = ExperimentAdmin(
-    model=Experiment, admin_site=AdminSite)
-
 
 class TestAdminExperiment(TestCase):
 
@@ -47,6 +44,11 @@ def setUpTestData(cls):
             session=Session.objects.first()
         )
 
+    def setUp(self):
+        self.admin = ExperimentAdmin(model=Experiment,
+                                     admin_site=AdminSite
+                                     )
+
     def test_experiment_model_fields(self):
         experiment = model_to_dict(Experiment.objects.first())
         experiment_fields = [key for key in experiment]
@@ -109,9 +111,12 @@ def setUpTestData(cls):
 
     def setUp(self):
         self.client = Client()
-
+        self.admin = ExperimentAdmin(model=Experiment,
+                                     admin_site=AdminSite
+                                     )
+
     def test_admin_export(self):
-        response = this_experiment_admin.export(request, self.experiment)
+        response = self.admin.export(request, self.experiment)
         zip_buffer = BytesIO(response.content)
         with ZipFile(zip_buffer, 'r') as test_zip:
             # Test files inside zip
@@ -174,8 +179,11 @@ def setUpTestData(self):
             description='test description very long like the tea of oolong and the song of the bird in the morning',
             slug='TEST',
         )
-        self.site = AdminSite()
-        self.admin = ExperimentCollectionAdmin(ExperimentCollection, self.site)
+
+    def setUp(self):
+        self.admin = ExperimentCollectionAdmin(model=ExperimentCollection,
+                                               admin_site=AdminSite
+                                               )
 
     def test_experiment_series_admin_list_display(self):
         self.assertEqual(
@@ -208,20 +216,16 @@ def test_experiment_collection_admin_research_dashboard(self):
 
 
 class PhaseAdminTest(TestCase):
-    @classmethod
-    def setUpTestData(self):
-        self.factory = RequestFactory()
-        self.site = AdminSite()
-        self.admin = PhaseAdmin(
-            Phase,
-            self.site
-        )
+
+    def setUp(self):
+        self.admin = PhaseAdmin(model=Phase,
+                                admin_site=AdminSite
+                                )
 
     def test_related_series_with_series(self):
         series = ExperimentCollection.objects.create(name='Test Series')
         phase = Phase.objects.create(
             name='Test Group', order=1, randomize=False, series=series, dashboard=True)
-        request = self.factory.get('/')
         related_series = self.admin.related_series(phase)
         expected_url = reverse(
             "admin:experiment_experimentcollection_change", args=[series.pk])
@@ -243,8 +247,7 @@ def test_experiments_with_experiments(self):
         experiment2 = Experiment.objects.create(name='Experiment 2', slug='experiment-2')
         grouped_experiment1 = GroupedExperiment.objects.create(phase=phase, experiment=experiment1)
         grouped_experiment2 = GroupedExperiment.objects.create(phase=phase, experiment=experiment2)
-
-        request = self.factory.get('/')
+
         experiments = self.admin.experiments(phase)
         expected_experiments = format_html(
             ', '.join([

backend/image/tests/test_admin.py

@@ -7,11 +7,11 @@
 
 
 class ImageAdminTest(TestCase):
-    @classmethod
-    def setUpTestData(cls):
-        cls.factory = RequestFactory()
-        cls.site = AdminSite()
-        cls.admin = ImageAdmin(Image, cls.site)
+
+    def setUp(self):
+        self.admin = ImageAdmin(model=Image,
+                                admin_site=AdminSite
+                                )
 
     def test_image_preview_with_file(self):
         image = Image.objects.create(file='path/to/image.jpg')

backend/requirements.in/base.txt

@@ -2,7 +2,7 @@
 audioread
 
 # Django Framework
-Django<4.0
+Django~=4.2.0
 
 # Django inline actions, admin extension
 django-inline-actions
@@ -18,7 +18,7 @@ python-dotenv
 IPToCC
 
 # PostgrSQL database client
-psycopg2
+psycopg[binary]
 
 # to convert labels to Roman numerals
 roman