Podmanify MUSCLE applications to enable automatic CI/CD (#722)

* config(docker): Add Docker Compose file for production environment and update frontend Dockerfile

* chore: Rename docker-compose file for deployment

* ci: Add Podman build & deploy workflow

* fix: Update Podman image build and deployment configuration path

* config(docker): Add ip2country service to docker-compose-deploy.yml

* ci: Use actions/checkout@v4 instead of v3

* ci(podman): Configure db service with bind mounts

* ci: Use test environment variables in podman workflow for now

* ci: Add server configuration

* ci: Check podman images after deploy

* ci: Update podman-compose command to force recreate containers

* ci: Add check for DJANGO_SETTINGS_MODULE environment variable

* ci: Add environment variables and secrets to Podman workflow

* ci: Add environment variable checks and log Podman images***

* ci: Add SQL_PORT environment variable

* ci: Update environment variables in podman.yml and docker-compose-deploy.yml

* ci: Use DockerfileDevelop for server for now

* ci: Update Podman workflow: Check logs of server service container after waiting 5 seconds

* config(docker): Add test target to DockerfileDevelop and docker-compose-deploy.yml

* ci: Update Podman workflow to check logs using podman-compose

* config(docker): Update DockerfileDevelop to use python:3.8 as base image

* ci: Refactor docker-compose-deploy.yml to run manage.py commands separately

* ci: Add Django superuser credentials to environment variables

* ci: Refactor command in docker-compose-deploy.yml

* ci: Add check for Django superuser environment variables

* config(docker): Install devtools if AML_DEBUG is set to true

* ci: Split up the backend's deploy commands

* ci: Use correct container name

* revert: Re-combine commands

* config: Add conditional installation of django-debug-toolbar in Dockerfile

* chore(deps): Add django-debug-toolbar to base.txt for development and test server

* ci: Add Podman check status workflow

* ci: Add healthcheck start_period to docker-compose file

* chore(deps): Move django-debug-toolbar back to development requirements

chore(deps): Remove django-debug-toolbar from requirements files

chore(deps): Add django-debug-toolbar for development

* config(docker): Add AML_DEBUG environment variable to docker-compose file

* ci: Add empty string for RUNNER_TRACKING_ID

* ci: Add .env file and break cache in Dockerfile

* ci: Add subpath support

* config: Add nginx-proxy service and custom-nginx.conf file

* config: Remove muscle client container

* config: Refactor proxy_pass configuration in custom-nginx.conf

* ci: Remove check status workflow

* config: Remove unnecessary settings related to static and media roots and cookie paths

* fix: Fix proxy URL in custom-nginx.conf

* config: Update nginx configuration to proxy requests to Django app

* config: Remove unnecessary commands and update Django settings and nginx configuration

* config: Update proxy_pass URL in custom-nginx.conf

* config: Add server dependency to docker-compose-deploy.yml

* config: Update static file URLs in production settings and nginx config

* config: Add proxy_redirect off directive to custom-nginx.conf

* config: Add subpath for running Django on /server

* config: Commented out root location in custom-nginx.conf

* config: Add proxy configuration for server

* config: Update Django configuration to remove subpath /server/

* config: Add proxy configuration for /server/ route

* config: Add URL prefix for server endpoints

* config: Update nginx configuration to serve frontend files for root requests

* docs: Add comment to STATIC_URL in production_settings.py

* ci: Add some documentation & rename job to deploy-test

* config(nginx): Use root instead of alias

* config(nginx): Mount django static assets & uploads directly in nginx folder

* refactor: Update dev.txt with django-debug-toolbar

* config: Add AML_SUBPATH environment variable

* ci: Update runs-on value from "self-hosted" to "tst" in podman.yml

* ci: Run deploy-test job on develop branch only

* ci: Add concurrency group for workflow on develop branch

* docker: Fetch Node.js 18 alpine image explicitly from docker.io

.github/workflows/podman.yml

@@ -0,0 +1,65 @@
+name: Podman build & deploy
+
+on:
+  push:
+    branches:
+      - develop
+  workflow_dispatch:
+
+jobs:
+  deploy-test:
+    name: Deploy to test environment
+    environment: test
+    runs-on: tst
+    if: github.ref == 'refs/heads/develop'
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+    env:
+
+      # Variables
+      AML_ALLOWED_HOSTS: ${{ vars.AML_ALLOWED_HOSTS }}
+      AML_CORS_ORIGIN_WHITELIST: ${{ vars.AML_CORS_ORIGIN_WHITELIST }}
+      AML_DEBUG: ${{ vars.AML_DEBUG }}
+      AML_LOCATION_PROVIDER: ${{ vars.AML_LOCATION_PROVIDER }}
+      AML_SUBPATH: ${{ vars.AML_SUBPATH }}
+      DJANGO_SETTINGS_MODULE: ${{ vars.DJANGO_SETTINGS_MODULE }}
+      SQL_DATABASE: ${{ vars.SQL_DATABASE }}
+      SQL_HOST: ${{ vars.SQL_HOST }}
+      SQL_PORT: ${{ vars.SQL_PORT }}
+      REACT_APP_API_ROOT: ${{ vars.REACT_APP_API_ROOT }}
+      REACT_APP_EXPERIMENT_SLUG: ${{ vars.REACT_APP_EXPERIMENT_SLUG }}
+      REACT_APP_AML_HOME: ${{ vars.REACT_APP_AML_HOME }}
+      REACT_APP_HTML_PAGE_TITLE: ${{ vars.REACT_APP_HTML_PAGE_TITLE }}
+
+      # Secrets
+      AML_SECRET_KEY: ${{ secrets.AML_SECRET_KEY }}
+      SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+      SQL_USER: ${{ secrets.SQL_USER }}
+      SQL_PASSWORD: ${{ secrets.SQL_PASSWORD }}
+      REACT_APP_SENTRY_DSN: ${{ secrets.REACT_APP_SENTRY_DSN }}
+      DJANGO_SUPERUSER_USERNAME: ${{ secrets.DJANGO_SUPERUSER_USERNAME }}
+      DJANGO_SUPERUSER_PASSWORD: ${{ secrets.DJANGO_SUPERUSER_PASSWORD }}
+      DJANGO_SUPERUSER_EMAIL: ${{ secrets.DJANGO_SUPERUSER_EMAIL }}
+
+      # Prevent podman services from exiting after startup
+      RUNNER_TRACKING_ID: ""
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create .env file
+        run: |
+          touch .env
+          echo "REACT_APP_API_ROOT=$REACT_APP_API_ROOT" >> .env
+          echo "REACT_APP_EXPERIMENT_SLUG=$REACT_APP_EXPERIMENT_SLUG" >> .env
+          echo "REACT_APP_AML_HOME=$REACT_APP_AML_HOME" >> .env
+          echo "REACT_APP_HTML_PAGE_TITLE=$REACT_APP_HTML_PAGE_TITLE" >> .env
+          echo "REACT_APP_SENTRY_DSN=$REACT_APP_SENTRY_DSN" >> .env
+          cp .env frontend/.env
+      - name: Build Podman images
+        run: podman-compose -f docker-compose-deploy.yml build
+      - name: Deploy Podman images
+        run: podman-compose -f docker-compose-deploy.yml up -d --force-recreate
+      - name: Check Podman images
+        run: podman-compose -f docker-compose-deploy.yml ps
+      - name: Check logs
+        run: podman-compose -f docker-compose-deploy.yml logs

backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8
+FROM docker.io/python:3.8
 ENV PYTHONUNBUFFERED 1
 RUN apt-get -y update
 RUN apt-get install -y ffmpeg
@@ -7,6 +7,6 @@ RUN apt-get install -y gettext
 WORKDIR /server
 COPY requirements/prod.txt /server/
 RUN pip install -r prod.txt
-# We add remainig code later, so pip install won't need to rerun if source code changes
-COPY . /server/
 
+# We add remainig code later, so pip install won't need to rerun if source code changes
+COPY . /server/

backend/DockerfileDevelop

@@ -1,9 +1,8 @@
-FROM python:3.8
+FROM docker.io/python:3.8 as base
 ENV PYTHONUNBUFFERED 1
 RUN apt-get -y update
 RUN apt-get install -y ffmpeg
 
 WORKDIR /server
 COPY requirements/dev.txt /server/
-RUN pip install -r dev.txt
-
+RUN pip install -r dev.txt

backend/aml/base_settings.py

@@ -176,3 +176,5 @@
     # We recommend adjusting this value in production.
     profiles_sample_rate=0.2,
 )
+
+SUBPATH = os.getenv('AML_SUBPATH', None)

backend/aml/production_settings.py

@@ -6,6 +6,8 @@
 # Database
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
 
+# Static url is set to /django_static/ in the nginx configuration
+# to avoid conflicts with the frontend's static files in /static/
 STATIC_URL = '/django_static/'
 
 DATABASES = {

backend/aml/urls.py

@@ -40,6 +40,11 @@
 #   ^ The static helper function only works in debug mode
 # (https://docs.djangoproject.com/en/3.0/howto/static-files/)
 
+
+# Prefix all URLS with /server if AML_SUBPATH is set
+if settings.SUBPATH:
+    urlpatterns = [path('server/', include(urlpatterns))]
+
 # Debug toolbar
 if settings.DEBUG:
     import debug_toolbar

backend/requirements/dev.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile --output-file=requirements/dev.txt requirements.in/dev.txt
 #
@@ -36,7 +36,7 @@ django==3.2.24
     #   django-inline-actions
 django-cors-headers==3.10.0
     # via -r requirements.in/base.txt
-django-debug-toolbar==3.2.2
+django-debug-toolbar==4.3.0
     # via -r requirements.in/dev.txt
 django-inline-actions==2.4.0
     # via -r requirements.in/base.txt
@@ -102,6 +102,7 @@ requests==2.31.0
     #   -r requirements.in/dev.txt
     #   genbadge
 roman==4.1
+    # via -r requirements.in/base.txt
 sentry-sdk==1.38.0
     # via -r requirements.in/base.txt
 six==1.16.0

backend/requirements/prod.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile --output-file=requirements/prod.txt requirements.in/prod.txt
 #
@@ -56,6 +56,7 @@ pytz==2023.3
 requests==2.31.0
     # via genbadge
 roman==4.1
+    # via -r requirements.in/base.txt
 sentry-sdk==1.38.0
     # via -r requirements.in/base.txt
 six==1.16.0

docker-compose-deploy.yml

@@ -0,0 +1,108 @@
+version: '3.8'
+
+services:
+  db:
+    image: postgres
+    environment:
+      - POSTGRES_DB=${SQL_DATABASE}
+      - POSTGRES_USER=${SQL_USER}
+      - POSTGRES_PASSWORD=${SQL_PASSWORD}
+      - PGHOST=${SQL_HOST}
+      - PGPORT=${SQL_PORT}
+      - PGUSER=${SQL_USER}
+      - PGDATABASE=${SQL_DATABASE}
+      - PGPASSWORD=${SQL_PASSWORD}
+    volumes:
+      - /home/github-runner/podman-volumes/db-data:/var/lib/postgresql/data
+      - /home/github-runner/podman-volumes/db-backups:/backups
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      start_period: 5s
+      interval: 5s
+      timeout: 5s
+      retries: 5
+  ip2country:
+    image: extrawurst/ip2country:latest
+    ports:
+      - 8854:5000
+  server:
+    build:
+      context: ./backend
+      dockerfile: Dockerfile
+    depends_on:
+      db:
+        condition: service_healthy
+      ip2country:
+        condition: service_started
+    volumes:
+      - /home/github-runner/podman-volumes/server-static:/server/static
+      - /home/github-runner/podman-volumes/server-logs:/server/logs
+      - /home/github-runner/podman-volumes/server-uploads:/server/upload
+    environment:
+      - AML_ALLOWED_HOSTS=${AML_ALLOWED_HOSTS}
+      - AML_DEBUG=${AML_DEBUG}
+      - AML_CORS_ORIGIN_WHITELIST=${AML_CORS_ORIGIN_WHITELIST}
+      - AML_LOCATION_PROVIDER=${AML_LOCATION_PROVIDER}
+      - AML_SECRET_KEY=${AML_SECRET_KEY}
+      - AML_SUBPATH=${AML_SUBPATH}
+      - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE}
+      - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME}
+      - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL}
+      - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD}
+      - SENTRY_DSN=${SENTRY_DSN}
+      - SQL_DATABASE=${SQL_DATABASE}
+      - SQL_USER=${SQL_USER}
+      - SQL_PASSWORD=${SQL_PASSWORD}
+      - SQL_HOST=${SQL_HOST}
+    ports:
+      - 8000:8000
+    command: bash -c "python manage.py migrate && python manage.py bootstrap && python manage.py collectstatic --noinput && gunicorn aml.wsgi:application --bind 0.0.0.0:8000"
+  client-builder:
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile
+      target: builder
+    volumes:
+      - type: bind
+        source: ./frontend/src
+        target: /client/src
+      - type: bind
+        source: ./frontend/public
+        target: /client/public
+      - type: bind
+        source: ./frontend/.storybook
+        target: /client/.storybook
+    environment:
+      - REACT_APP_API_ROOT=${REACT_APP_API_ROOT}
+      - REACT_APP_EXPERIMENT_SLUG=${REACT_APP_EXPERIMENT_SLUG}
+      - REACT_APP_AML_HOME=${REACT_APP_AML_HOME}
+      - REACT_APP_LOGO_URL=${REACT_APP_LOGO_URL}
+      - REACT_APP_HTML_FAVICON=${REACT_APP_HTML_FAVICON}
+      - REACT_APP_HTML_PAGE_TITLE=${REACT_APP_HTML_PAGE_TITLE}
+      - REACT_APP_HTML_OG_DESCRIPTION=${REACT_APP_HTML_OG_DESCRIPTION}
+      - REACT_APP_HTML_OG_IMAGE=${REACT_APP_HTML_OG_IMAGE}
+      - REACT_APP_HTML_OG_TITLE=${REACT_APP_HTML_OG_TITLE}
+      - REACT_APP_HTML_OG_URL=${REACT_APP_HTML_OG_URL}
+      - REACT_APP_HTML_BODY_CLASS=${REACT_APP_HTML_BODY_CLASS}
+      - REACT_APP_SENTRY_DSN=${REACT_APP_SENTRY_DSN}
+      - REACT_APP_STRICT=${REACT_APP_STRICT}
+
+  # This service is responsible for serving 
+  # 1. The built frontend from the client-builder service
+  # 2. The static files from the server service (e.g., static files & uploads from the Django app)
+  nginx-proxy:
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile
+      target: runner
+    command: nginx -g 'daemon off;'
+    ports:
+      - 3000:80
+    depends_on:
+      - client-builder
+      - server
+    volumes:
+      - /home/github-runner/podman-volumes/server-static:/usr/share/nginx/html/django_static
+      - /home/github-runner/podman-volumes/server-uploads:/usr/share/nginx/html/upload
+      - ./nginx/custom-nginx.conf:/etc/nginx/conf.d/default.conf
+

frontend/Dockerfile

@@ -1,9 +1,22 @@
-FROM node:18-alpine
+# Builder image
+FROM docker.io/node:18-alpine as builder
 
 RUN yarn set version stable
 WORKDIR /client
 COPY package.json /client/
 COPY yarn.lock /client/
 COPY jsconfig.json /client/
-RUN yarn
 COPY . /client/
+
+# Copy .env file to break the cache
+COPY .env /client/
+
+RUN yarn
+RUN yarn scss
+RUN yarn build
+
+# Runner image that serves the built app using nginx
+FROM docker.io/nginx:alpine as runner
+
+COPY --from=builder /client/build /usr/share/nginx/html
+EXPOSE 80

nginx/custom-nginx.conf

@@ -0,0 +1,18 @@
+server {
+    listen 80;
+
+    # Serve frontend files for root requests
+    location / {
+        root /usr/share/nginx/html;
+        try_files $uri $uri/ /index.html;
+    }
+
+    # Proxy pass to the Django app
+    location /server/ {
+        proxy_pass http://server:8000;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+}