From 2b4f2bd1bbc8499577541bd5e2af735459d431f7 Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Mon, 19 Feb 2024 13:43:12 +0100 Subject: [PATCH] Podmanify MUSCLE applications to enable automatic CI/CD (#722) * config(docker): Add Docker Compose file for production environment and update frontend Dockerfile * chore: Rename docker-compose file for deployment * ci: Add Podman build & deploy workflow * fix: Update Podman image build and deployment configuration path * config(docker): Add ip2country service to docker-compose-deploy.yml * ci: Use actions/checkout@v4 instead of v3 * ci(podman): Configure db service with bind mounts * ci: Use test environment variables in podman workflow for now * ci: Add server configuration * ci: Check podman images after deploy * ci: Update podman-compose command to force recreate containers * ci: Add check for DJANGO_SETTINGS_MODULE environment variable * ci: Add environment variables and secrets to Podman workflow * ci: Add environment variable checks and log Podman images*** * ci: Add SQL_PORT environment variable * ci: Update environment variables in podman.yml and docker-compose-deploy.yml * ci: Use DockerfileDevelop for server for now * ci: Update Podman workflow: Check logs of server service container after waiting 5 seconds * config(docker): Add test target to DockerfileDevelop and docker-compose-deploy.yml * ci: Update Podman workflow to check logs using podman-compose * config(docker): Update DockerfileDevelop to use python:3.8 as base image * ci: Refactor docker-compose-deploy.yml to run manage.py commands separately * ci: Add Django superuser credentials to environment variables * ci: Refactor command in docker-compose-deploy.yml * ci: Add check for Django superuser environment variables * config(docker): Install devtools if AML_DEBUG is set to true * ci: Split up the backend's deploy commands * ci: Use correct container name * revert: Re-combine commands * config: Add conditional installation of django-debug-toolbar in Dockerfile * chore(deps): Add django-debug-toolbar to base.txt for development and test server * ci: Add Podman check status workflow * ci: Add healthcheck start_period to docker-compose file * chore(deps): Move django-debug-toolbar back to development requirements chore(deps): Remove django-debug-toolbar from requirements files chore(deps): Add django-debug-toolbar for development * config(docker): Add AML_DEBUG environment variable to docker-compose file * ci: Add empty string for RUNNER_TRACKING_ID * ci: Add .env file and break cache in Dockerfile * ci: Add subpath support * config: Add nginx-proxy service and custom-nginx.conf file * config: Remove muscle client container * config: Refactor proxy_pass configuration in custom-nginx.conf * ci: Remove check status workflow * config: Remove unnecessary settings related to static and media roots and cookie paths * fix: Fix proxy URL in custom-nginx.conf * config: Update nginx configuration to proxy requests to Django app * config: Remove unnecessary commands and update Django settings and nginx configuration * config: Update proxy_pass URL in custom-nginx.conf * config: Add server dependency to docker-compose-deploy.yml * config: Update static file URLs in production settings and nginx config * config: Add proxy_redirect off directive to custom-nginx.conf * config: Add subpath for running Django on /server * config: Commented out root location in custom-nginx.conf * config: Add proxy configuration for server * config: Update Django configuration to remove subpath /server/ * config: Add proxy configuration for /server/ route * config: Add URL prefix for server endpoints * config: Update nginx configuration to serve frontend files for root requests * docs: Add comment to STATIC_URL in production_settings.py * ci: Add some documentation & rename job to deploy-test * config(nginx): Use root instead of alias * config(nginx): Mount django static assets & uploads directly in nginx folder * refactor: Update dev.txt with django-debug-toolbar * config: Add AML_SUBPATH environment variable * ci: Update runs-on value from "self-hosted" to "tst" in podman.yml * ci: Run deploy-test job on develop branch only * ci: Add concurrency group for workflow on develop branch * docker: Fetch Node.js 18 alpine image explicitly from docker.io --- .github/workflows/podman.yml | 65 +++++++++++++++++ backend/Dockerfile | 6 +- backend/DockerfileDevelop | 5 +- backend/aml/base_settings.py | 2 + backend/aml/production_settings.py | 2 + backend/aml/urls.py | 5 ++ backend/requirements/dev.txt | 7 +- backend/requirements/prod.txt | 5 +- docker-compose-deploy.yml | 108 +++++++++++++++++++++++++++++ frontend/Dockerfile | 17 ++++- nginx/custom-nginx.conf | 18 +++++ 11 files changed, 227 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/podman.yml create mode 100644 docker-compose-deploy.yml create mode 100644 nginx/custom-nginx.conf diff --git a/.github/workflows/podman.yml b/.github/workflows/podman.yml new file mode 100644 index 000000000..cb302785a --- /dev/null +++ b/.github/workflows/podman.yml @@ -0,0 +1,65 @@ +name: Podman build & deploy + +on: + push: + branches: + - develop + workflow_dispatch: + +jobs: + deploy-test: + name: Deploy to test environment + environment: test + runs-on: tst + if: github.ref == 'refs/heads/develop' + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + env: + + # Variables + AML_ALLOWED_HOSTS: ${{ vars.AML_ALLOWED_HOSTS }} + AML_CORS_ORIGIN_WHITELIST: ${{ vars.AML_CORS_ORIGIN_WHITELIST }} + AML_DEBUG: ${{ vars.AML_DEBUG }} + AML_LOCATION_PROVIDER: ${{ vars.AML_LOCATION_PROVIDER }} + AML_SUBPATH: ${{ vars.AML_SUBPATH }} + DJANGO_SETTINGS_MODULE: ${{ vars.DJANGO_SETTINGS_MODULE }} + SQL_DATABASE: ${{ vars.SQL_DATABASE }} + SQL_HOST: ${{ vars.SQL_HOST }} + SQL_PORT: ${{ vars.SQL_PORT }} + REACT_APP_API_ROOT: ${{ vars.REACT_APP_API_ROOT }} + REACT_APP_EXPERIMENT_SLUG: ${{ vars.REACT_APP_EXPERIMENT_SLUG }} + REACT_APP_AML_HOME: ${{ vars.REACT_APP_AML_HOME }} + REACT_APP_HTML_PAGE_TITLE: ${{ vars.REACT_APP_HTML_PAGE_TITLE }} + + # Secrets + AML_SECRET_KEY: ${{ secrets.AML_SECRET_KEY }} + SENTRY_DSN: ${{ secrets.SENTRY_DSN }} + SQL_USER: ${{ secrets.SQL_USER }} + SQL_PASSWORD: ${{ secrets.SQL_PASSWORD }} + REACT_APP_SENTRY_DSN: ${{ secrets.REACT_APP_SENTRY_DSN }} + DJANGO_SUPERUSER_USERNAME: ${{ secrets.DJANGO_SUPERUSER_USERNAME }} + DJANGO_SUPERUSER_PASSWORD: ${{ secrets.DJANGO_SUPERUSER_PASSWORD }} + DJANGO_SUPERUSER_EMAIL: ${{ secrets.DJANGO_SUPERUSER_EMAIL }} + + # Prevent podman services from exiting after startup + RUNNER_TRACKING_ID: "" + + steps: + - uses: actions/checkout@v4 + - name: Create .env file + run: | + touch .env + echo "REACT_APP_API_ROOT=$REACT_APP_API_ROOT" >> .env + echo "REACT_APP_EXPERIMENT_SLUG=$REACT_APP_EXPERIMENT_SLUG" >> .env + echo "REACT_APP_AML_HOME=$REACT_APP_AML_HOME" >> .env + echo "REACT_APP_HTML_PAGE_TITLE=$REACT_APP_HTML_PAGE_TITLE" >> .env + echo "REACT_APP_SENTRY_DSN=$REACT_APP_SENTRY_DSN" >> .env + cp .env frontend/.env + - name: Build Podman images + run: podman-compose -f docker-compose-deploy.yml build + - name: Deploy Podman images + run: podman-compose -f docker-compose-deploy.yml up -d --force-recreate + - name: Check Podman images + run: podman-compose -f docker-compose-deploy.yml ps + - name: Check logs + run: podman-compose -f docker-compose-deploy.yml logs \ No newline at end of file diff --git a/backend/Dockerfile b/backend/Dockerfile index 4eda53084..3203cdf09 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8 +FROM docker.io/python:3.8 ENV PYTHONUNBUFFERED 1 RUN apt-get -y update RUN apt-get install -y ffmpeg @@ -7,6 +7,6 @@ RUN apt-get install -y gettext WORKDIR /server COPY requirements/prod.txt /server/ RUN pip install -r prod.txt -# We add remainig code later, so pip install won't need to rerun if source code changes -COPY . /server/ +# We add remainig code later, so pip install won't need to rerun if source code changes +COPY . /server/ \ No newline at end of file diff --git a/backend/DockerfileDevelop b/backend/DockerfileDevelop index bcecba141..139171da6 100644 --- a/backend/DockerfileDevelop +++ b/backend/DockerfileDevelop @@ -1,9 +1,8 @@ -FROM python:3.8 +FROM docker.io/python:3.8 as base ENV PYTHONUNBUFFERED 1 RUN apt-get -y update RUN apt-get install -y ffmpeg WORKDIR /server COPY requirements/dev.txt /server/ -RUN pip install -r dev.txt - +RUN pip install -r dev.txt \ No newline at end of file diff --git a/backend/aml/base_settings.py b/backend/aml/base_settings.py index 911894c8c..c8f80a353 100644 --- a/backend/aml/base_settings.py +++ b/backend/aml/base_settings.py @@ -176,3 +176,5 @@ # We recommend adjusting this value in production. profiles_sample_rate=0.2, ) + +SUBPATH = os.getenv('AML_SUBPATH', None) diff --git a/backend/aml/production_settings.py b/backend/aml/production_settings.py index 24bb4dfa4..f9ac05ee8 100644 --- a/backend/aml/production_settings.py +++ b/backend/aml/production_settings.py @@ -6,6 +6,8 @@ # Database # https://docs.djangoproject.com/en/3.0/ref/settings/#databases +# Static url is set to /django_static/ in the nginx configuration +# to avoid conflicts with the frontend's static files in /static/ STATIC_URL = '/django_static/' DATABASES = { diff --git a/backend/aml/urls.py b/backend/aml/urls.py index 3188931e7..9444450fc 100644 --- a/backend/aml/urls.py +++ b/backend/aml/urls.py @@ -40,6 +40,11 @@ # ^ The static helper function only works in debug mode # (https://docs.djangoproject.com/en/3.0/howto/static-files/) + +# Prefix all URLS with /server if AML_SUBPATH is set +if settings.SUBPATH: + urlpatterns = [path('server/', include(urlpatterns))] + # Debug toolbar if settings.DEBUG: import debug_toolbar diff --git a/backend/requirements/dev.txt b/backend/requirements/dev.txt index c0ceceb7e..c053b35fd 100644 --- a/backend/requirements/dev.txt +++ b/backend/requirements/dev.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=requirements/dev.txt requirements.in/dev.txt # @@ -36,7 +36,7 @@ django==3.2.24 # django-inline-actions django-cors-headers==3.10.0 # via -r requirements.in/base.txt -django-debug-toolbar==3.2.2 +django-debug-toolbar==4.3.0 # via -r requirements.in/dev.txt django-inline-actions==2.4.0 # via -r requirements.in/base.txt @@ -102,6 +102,7 @@ requests==2.31.0 # -r requirements.in/dev.txt # genbadge roman==4.1 + # via -r requirements.in/base.txt sentry-sdk==1.38.0 # via -r requirements.in/base.txt six==1.16.0 diff --git a/backend/requirements/prod.txt b/backend/requirements/prod.txt index 6dc51a490..d92f35173 100644 --- a/backend/requirements/prod.txt +++ b/backend/requirements/prod.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=requirements/prod.txt requirements.in/prod.txt # @@ -56,6 +56,7 @@ pytz==2023.3 requests==2.31.0 # via genbadge roman==4.1 + # via -r requirements.in/base.txt sentry-sdk==1.38.0 # via -r requirements.in/base.txt six==1.16.0 diff --git a/docker-compose-deploy.yml b/docker-compose-deploy.yml new file mode 100644 index 000000000..243f8bed4 --- /dev/null +++ b/docker-compose-deploy.yml @@ -0,0 +1,108 @@ +version: '3.8' + +services: + db: + image: postgres + environment: + - POSTGRES_DB=${SQL_DATABASE} + - POSTGRES_USER=${SQL_USER} + - POSTGRES_PASSWORD=${SQL_PASSWORD} + - PGHOST=${SQL_HOST} + - PGPORT=${SQL_PORT} + - PGUSER=${SQL_USER} + - PGDATABASE=${SQL_DATABASE} + - PGPASSWORD=${SQL_PASSWORD} + volumes: + - /home/github-runner/podman-volumes/db-data:/var/lib/postgresql/data + - /home/github-runner/podman-volumes/db-backups:/backups + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + start_period: 5s + interval: 5s + timeout: 5s + retries: 5 + ip2country: + image: extrawurst/ip2country:latest + ports: + - 8854:5000 + server: + build: + context: ./backend + dockerfile: Dockerfile + depends_on: + db: + condition: service_healthy + ip2country: + condition: service_started + volumes: + - /home/github-runner/podman-volumes/server-static:/server/static + - /home/github-runner/podman-volumes/server-logs:/server/logs + - /home/github-runner/podman-volumes/server-uploads:/server/upload + environment: + - AML_ALLOWED_HOSTS=${AML_ALLOWED_HOSTS} + - AML_DEBUG=${AML_DEBUG} + - AML_CORS_ORIGIN_WHITELIST=${AML_CORS_ORIGIN_WHITELIST} + - AML_LOCATION_PROVIDER=${AML_LOCATION_PROVIDER} + - AML_SECRET_KEY=${AML_SECRET_KEY} + - AML_SUBPATH=${AML_SUBPATH} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} + - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME} + - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL} + - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD} + - SENTRY_DSN=${SENTRY_DSN} + - SQL_DATABASE=${SQL_DATABASE} + - SQL_USER=${SQL_USER} + - SQL_PASSWORD=${SQL_PASSWORD} + - SQL_HOST=${SQL_HOST} + ports: + - 8000:8000 + command: bash -c "python manage.py migrate && python manage.py bootstrap && python manage.py collectstatic --noinput && gunicorn aml.wsgi:application --bind 0.0.0.0:8000" + client-builder: + build: + context: ./frontend + dockerfile: Dockerfile + target: builder + volumes: + - type: bind + source: ./frontend/src + target: /client/src + - type: bind + source: ./frontend/public + target: /client/public + - type: bind + source: ./frontend/.storybook + target: /client/.storybook + environment: + - REACT_APP_API_ROOT=${REACT_APP_API_ROOT} + - REACT_APP_EXPERIMENT_SLUG=${REACT_APP_EXPERIMENT_SLUG} + - REACT_APP_AML_HOME=${REACT_APP_AML_HOME} + - REACT_APP_LOGO_URL=${REACT_APP_LOGO_URL} + - REACT_APP_HTML_FAVICON=${REACT_APP_HTML_FAVICON} + - REACT_APP_HTML_PAGE_TITLE=${REACT_APP_HTML_PAGE_TITLE} + - REACT_APP_HTML_OG_DESCRIPTION=${REACT_APP_HTML_OG_DESCRIPTION} + - REACT_APP_HTML_OG_IMAGE=${REACT_APP_HTML_OG_IMAGE} + - REACT_APP_HTML_OG_TITLE=${REACT_APP_HTML_OG_TITLE} + - REACT_APP_HTML_OG_URL=${REACT_APP_HTML_OG_URL} + - REACT_APP_HTML_BODY_CLASS=${REACT_APP_HTML_BODY_CLASS} + - REACT_APP_SENTRY_DSN=${REACT_APP_SENTRY_DSN} + - REACT_APP_STRICT=${REACT_APP_STRICT} + + # This service is responsible for serving + # 1. The built frontend from the client-builder service + # 2. The static files from the server service (e.g., static files & uploads from the Django app) + nginx-proxy: + build: + context: ./frontend + dockerfile: Dockerfile + target: runner + command: nginx -g 'daemon off;' + ports: + - 3000:80 + depends_on: + - client-builder + - server + volumes: + - /home/github-runner/podman-volumes/server-static:/usr/share/nginx/html/django_static + - /home/github-runner/podman-volumes/server-uploads:/usr/share/nginx/html/upload + - ./nginx/custom-nginx.conf:/etc/nginx/conf.d/default.conf + \ No newline at end of file diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 984f171cf..0e7dd323e 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,9 +1,22 @@ -FROM node:18-alpine +# Builder image +FROM docker.io/node:18-alpine as builder RUN yarn set version stable WORKDIR /client COPY package.json /client/ COPY yarn.lock /client/ COPY jsconfig.json /client/ -RUN yarn COPY . /client/ + +# Copy .env file to break the cache +COPY .env /client/ + +RUN yarn +RUN yarn scss +RUN yarn build + +# Runner image that serves the built app using nginx +FROM docker.io/nginx:alpine as runner + +COPY --from=builder /client/build /usr/share/nginx/html +EXPOSE 80 \ No newline at end of file diff --git a/nginx/custom-nginx.conf b/nginx/custom-nginx.conf new file mode 100644 index 000000000..a87eefed9 --- /dev/null +++ b/nginx/custom-nginx.conf @@ -0,0 +1,18 @@ +server { + listen 80; + + # Serve frontend files for root requests + location / { + root /usr/share/nginx/html; + try_files $uri $uri/ /index.html; + } + + # Proxy pass to the Django app + location /server/ { + proxy_pass http://server:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +}