compose.yml

x-backend-common-envs: &backend-common-envs
  DB_NAME: eurydice
  DB_USER: eurydice
  DB_PASSWORD: eurydice
  DB_PORT: "5432"
  DJANGO_ENV: DEV
  DEBUG: "True"
  DJANGO_MANAGEPY_MIGRATE: "off"
  MINIO_ACCESS_KEY: minio
  MINIO_SECRET_KEY: miniominio
  TZ: "Europe/Paris"
  SECURE_COOKIES: "false"
  USER_ASSOCIATION_TOKEN_SECRET_KEY: "Baby shark for 36 bytes dudududududu"
  EURYDICE_CONTACT: ${EURYDICE_CONTACT:-the development team}
  EURYDICE_CONTACT_FR: ${EURYDICE_CONTACT_FR:-par courrier recommandé en 3 exemplaires}
  UI_BADGE_CONTENT: Dev -> Dev
  UI_BADGE_COLOR: "green"
  LOG_TO_FILE: true

x-backend: &backend
  build:
    context: "backend"
    dockerfile: docker/Dockerfile
    target: dev
  # NOTE: we tag local dev images with our registry otherwise the image is tagged
  # using the docker hub as a registry
  # https://docs.docker.com/engine/reference/commandline/tag/#extended-description
  image: eurydice/backend:dev
  networks:
    - eurydice
  command:
    - make
    - run-dev

x-common-origin-envs: &common-origin-envs
  DB_HOST: db-origin
  MINIO_ENABLED: true
  MINIO_ENDPOINT: minio:9000
  MINIO_BUCKET_NAME: eurydice-origin
  # Maximum object size per operation allowed by minio (50 TiB)
  # https://min.io/docs/minio/linux/operations/checklists/thresholds.html#minio-server-limits
  TRANSFERABLE_MAX_SIZE: 54975581388800

x-common-destination-envs: &common-destination-envs
  DB_HOST: db-destination
  MINIO_ENABLED: false
  TRANSFERABLE_STORAGE_DIR: /tmp/eurydice-data/
  MINIO_BUCKET_NAME: eurydice-destination

x-common-auth-envs: &common-auth-envs
  REMOTE_USER_HEADER_AUTHENTICATION_ENABLED: "true"

x-db: &db
  image: postgres:15
  read_only: true
  networks:
    - eurydice
  environment:
    POSTGRES_DB: eurydice
    POSTGRES_USER: eurydice
    POSTGRES_PASSWORD: eurydice
    TZ: "Europe/Paris"
    PGTZ: "Europe/Paris"
  tmpfs:
    - /tmp
    - /run
  command:
    - "postgres"
    - "-c" # turn on csv logging
    - "logging_collector=on"
    - "-c"
    - "log_destination=csvlog"
    - "-c"
    - "log_directory=/var/log/postgresql"
    - "-c" # filebeat format
    - "log_line_prefix=%m [%p] %q%u@%d "
    - "-c" # turn on hour-based log rotation
    - "log_truncate_on_rotation=on"
    - "-c"
    - "log_filename=postgresql-%H.log"
    - "-c"
    - "log_rotation_age=60"
    - "-c"
    - "log_file_mode=0644"
    - "-c" # print statements that took more than 200ms...
    - "log_min_duration_statement=200"
    - "-c" # ...do not print other statements...
    - "log_statement=none"
    - "-c" # ...do not even print the duration of these other statements
    - "log_duration=off"
  healthcheck:
    # NOTE: DB will become healthy as soon as it has created the eurydice table
    test:
      - "CMD-SHELL"
      - 'psql -U eurydice -lqtA | grep -q "^eurydice|"'
    interval: 5s
    timeout: 5s
    retries: 5

services:
  ################ Common Services ################

  traefik:
    image: traefik:2.10
    networks:
      - eurydice
    command:
      - "--log.level=DEBUG"
      - "--providers.docker=true"
      - "--providers.docker.exposedbydefault=false"
      - "--entrypoints.web.address=:80"
      - "--api.dashboard=true"
    ports:
      - "80:80"
    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
    labels:
      traefik.enable: "true"
      traefik.http.routers.dashboard.rule: Host(`traefik.localhost`)
      traefik.http.routers.dashboard.service: api@internal

  minio:
    image: minio/minio:RELEASE.2023-08-23T10-07-06Z
    networks:
      - eurydice
    environment:
      MINIO_ROOT_USER: minio
      MINIO_ROOT_PASSWORD: miniominio
    volumes:
      - minio-data:/data
      - minio-config:/config
    ports:
      - 9000:9000
    command:
      - server
      - /data
      - --config-dir
      - /config/
      - --certs-dir
      - /config/.certs/
    labels:
      traefik.enable: "true"
      traefik.http.routers.minio.rule: Host(`minio.localhost`)
    healthcheck:
      test:
        - "CMD-SHELL"
        - "curl http://127.0.0.1:9000/minio/health/live"
      interval: 5s
      timeout: 5s
      retries: 10
      start_period: 2s

  frontend:
    build:
      context: "frontend"
      dockerfile: docker/Dockerfile
      target: dev
    # note: we tag local dev images with our registry otherwise the image is tagged
    # using the docker hub as a registry
    # https://docs.docker.com/engine/reference/commandline/tag/#extended-description
    image: eurydice/frontend:dev
    networks:
      - eurydice
    volumes:
      - ./frontend:/home/eurydice/frontend
    command:
      - make
      - dev
    environment:
      NODE_ENV: development
      # workaround for vue-cli container detection not working, see:
      # https://github.com/vuejs/vue-cli/issues/6726
      CODESANDBOX_SSE: "1"
    labels:
      traefik.enable: "true"
      traefik.http.routers.frontend.rule: Host(`origin.localhost`) || Host(`destination.localhost`)
      traefik.http.routers.frontend.priority: "1"

  ################ Origin Services ################

  backend-origin:
    <<: *backend
    depends_on:
      db-origin:
        condition: service_started
      minio:
        condition: service_healthy
    volumes:
      - ./backend:/home/eurydice/backend
      - backend-origin-logs:/var/log/app
    environment:
      <<: [*backend-common-envs, *common-origin-envs, *common-auth-envs]
      SECRET_KEY: GhSZiXkmYyty
      EURYDICE_API: origin
      ALLOWED_HOSTS: origin.localhost,backend-origin
      CSRF_TRUSTED_ORIGINS: ""
      DJANGO_MANAGEPY_MIGRATE: "on"
      METRICS_SLIDING_WINDOW: ${METRICS_SLIDING_WINDOW:-60min}
    healthcheck:
      test:
        - "CMD"
        - /healthcheck.py
      interval: 5s
      timeout: 5s
      retries: 5
      # NOTE: in the dev image the stat period is longer because we expect
      #       the container to apply migrations on first startup
      #       (in production this should be done by a different service)
      start_period: 5s
    labels:
      traefik.enable: "true"
      traefik.http.routers.backend-origin.rule: Host(`origin.localhost`) && PathPrefix(`/api`, `/admin`, `/static`)
    command:
      - make
      - run-origin-api

  sender:
    <<: *backend
    depends_on:
      # NOTE: we await for the backend container to be healthy
      #       because it is the one runnning the migrations in
      #       the developement environment
      backend-origin:
        condition: service_healthy
      minio:
        condition: service_healthy
      lidis:
        condition: service_started
    volumes:
      - ./backend:/home/eurydice/backend
      - sender-logs:/var/log/app
    extra_hosts:
      - "lidis.docker.host:${LIDIS_DOCKER_HOST:-host-gateway}"
    environment:
      <<: [*backend-common-envs, *common-origin-envs]
      DB_HOST: db-origin
      MINIO_BUCKET_NAME: eurydice-origin
      LIDIS_HOST: "lidis.docker.host"
      LIDIS_PORT: "5000"
      SENDER_RANGE_FILLER_CLASS: "UserRotatingTransferableRangeFiller"
    command:
      - make
      - run-sender

  dbtrimmer-origin:
    <<: *backend
    depends_on:
      # NOTE: we await for the backend container to be healthy
      #       because it is the one runnning the migrations in
      #       the developement environment
      backend-origin:
        condition: service_healthy
    volumes:
      - ./backend:/home/eurydice/backend
      - dbtrimmer-origin-logs:/var/log/app
    environment:
      <<: [*backend-common-envs, *common-origin-envs]
      DBTRIMMER_TRIM_TRANSFERABLES_AFTER: "2h"
      DBTRIMMER_RUN_EVERY: "6h"
      DBTRIMMER_POLL_EVERY: "200ms"
    command:
      - make
      - run-origin-dbtrimmer

  db-origin:
    <<: *db
    ports:
      - 5432:5432
    volumes:
      - db-origin-data:/var/lib/postgresql/data
      - db-origin-logs:/var/log/postgresql/:rw

  lidis:
    image: eurydice/lidi-send:latest
    network_mode: "host"
    environment:
      RUST_LOG: ${RUST_LOG:-INFO}
    depends_on:
      lidir:
        condition: service_started
    read_only: true
    command:
      - "--from_tcp"
      - "0.0.0.0:5000"
      - "--to_udp"
      - "0.0.0.0:6000"
      - "--nb_clients"
      - "1"

  ################ Destination Services ################

  lidir:
    image: eurydice/lidi-receive:latest
    network_mode: "host"
    environment:
      RUST_LOG: ${RUST_LOG:-INFO}
    depends_on:
      receiver:
        condition: service_started
    read_only: true
    command:
      - "--from_udp"
      - "0.0.0.0:6000"
      - "--to_tcp"
      - "0.0.0.0:7000"
      - "--nb_clients"
      - "1"

  db-destination:
    <<: *db
    ports:
      - 5433:5432
    volumes:
      - db-destination-data:/var/lib/postgresql/data
      - db-destination-logs:/var/log/postgresql/

  receiver:
    <<: *backend
    depends_on:
      # NOTE: we await for the backend container to be healthy
      #       because it is the one runnning the migrations in
      #       the developement environment
      backend-destination:
        condition: service_healthy
      minio:
        condition: service_healthy
    volumes:
      - ./backend:/home/eurydice/backend
      - receiver-logs:/var/log/app
      - destination-data-storage:/tmp/eurydice-data
    environment:
      <<: [*backend-common-envs, *common-destination-envs]
      PACKET_RECEIVER_HOST: "0.0.0.0"
      PACKET_RECEIVER_PORT: "7000"
      RECEIVER_BUFFER_MAX_ITEMS: "4"
    ports:
      - "127.0.0.1:7000:7000"
    command:
      - make
      - run-receiver

  dbtrimmer-destination:
    <<: *backend
    depends_on:
      # NOTE: we await for the backend container to be healthy
      #       because it is the one runnning the migrations in
      #       the developement environment
      backend-destination:
        condition: service_healthy
    volumes:
      - ./backend:/home/eurydice/backend
      - dbtrimmer-destination-logs:/var/log/app
      - destination-data-storage:/tmp/eurydice-data
    environment:
      <<: [*backend-common-envs, *common-destination-envs]
      DBTRIMMER_TRIM_TRANSFERABLES_AFTER: "2h"
      DBTRIMMER_RUN_EVERY: "6h"
      DBTRIMMER_POLL_EVERY: "200ms"
    command:
      - make
      - run-destination-dbtrimmer

  s3remover:
    <<: *backend
    depends_on:
      # NOTE: we await for the backend container to be healthy
      #       because it is the one runnning the migrations in
      #       the developement environment
      backend-destination:
        condition: service_healthy
      minio:
        condition: service_healthy
    volumes:
      - ./backend:/home/eurydice/backend
      - s3remover-logs:/var/log/app
      - destination-data-storage:/tmp/eurydice-data
    environment:
      <<: [*backend-common-envs, *common-destination-envs]
      S3REMOVER_EXPIRE_TRANSFERABLES_AFTER: "1hour"
      S3REMOVER_RUN_EVERY: "30s"
      S3REMOVER_POLL_EVERY: "200ms"
    command:
      - make
      - run-destination-s3remover

  backend-destination:
    <<: *backend
    depends_on:
      db-destination:
        condition: service_healthy
      minio:
        condition: service_healthy
    volumes:
      - ./backend:/home/eurydice/backend
      - backend-destination-logs:/var/log/app
      - destination-data-storage:/tmp/eurydice-data
    environment:
      <<: [*backend-common-envs, *common-destination-envs, *common-auth-envs]
      SECRET_KEY: jBS0nmi4MOQIeXcN
      EURYDICE_API: destination
      ALLOWED_HOSTS: destination.localhost,backend-destination
      CSRF_TRUSTED_ORIGINS: ""
      DJANGO_MANAGEPY_MIGRATE: "on"
      METRICS_SLIDING_WINDOW: ${METRICS_SLIDING_WINDOW:-60min}
    healthcheck:
      test:
        - CMD
        - /healthcheck.py
      interval: 5s
      timeout: 5s
      retries: 5
      # NOTE: in the dev image the stat period is longer because we expect
      #       the container to apply migrations on first startup
      #       (in production this should be done by a different service)
      start_period: 5s
    labels:
      traefik.enable: "true"
      traefik.http.routers.backend-destination.rule: Host(`destination.localhost`) && PathPrefix(`/api`, `/admin`, `/static`)
    command:
      - make
      - run-destination-api

  ################ Dev Tools ################

  pgadmin:
    build:
      context: pgadmin
      target: dev
    # NOTE: we tag local dev images with our registry otherwise the image is tagged
    # using the docker hub as a registry
    # https://docs.docker.com/engine/reference/commandline/tag/#extended-description
    image: eurydice/pgadmin4:dev
    environment:
      PGADMIN_DEFAULT_EMAIL: admin@example.com
      PGADMIN_DEFAULT_PASSWORD: admin
      PGADMIN_CONFIG_SERVER_MODE: "False"
      PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED: "False"
    networks:
      - eurydice
    volumes:
      - pgadmin:/var/lib/pgadmin
    labels:
      traefik.enable: "true"
      traefik.http.routers.pgadmin.rule: Host(`pgadmin.localhost`)

volumes:
  db-origin-data:
  db-origin-logs:
  db-destination-data:
  db-destination-logs:
  minio-data:
  minio-config:
  pgadmin:
  backend-origin-logs:
  sender-logs:
  dbtrimmer-origin-logs:
  backend-destination-logs:
  receiver-logs:
  dbtrimmer-destination-logs:
  s3remover-logs:
  destination-data-storage:

networks:
  eurydice:
    driver: bridge
    name: eurydice
    ipam:
      config:
        - subnet: ${EURYDICE_IPV4_SUBNET:-172.18.0.0/24}