docker-compose.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# Basic Airflow cluster configuration for CeleryExecutor with Redis and PostgreSQL.
#
# WARNING: This configuration is for local development. Do not use it in a production deployment.
#
# This configuration supports basic configuration using environment variables or an .env file
# See README.md > Environment variable
#
# Feel free to modify this file to suit your needs.
---
version: '3'

x-airflow-common:
  # In order to add custom dependencies or upgrade provider packages you can use your extended image.
  # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml
  # and uncomment the "build" line below, Then run `docker-compose build` to build the images.
  &airflow-common
  image: importer/airflow
  build:
    context: ./
    dockerfile: ./docker/Dockerfile
    args:
      CUSTOM_UID: ${AIRFLOW_UID}
  environment:
    &airflow-common-env
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER:-airflow}:${POSTGRES_PASSWORD:-airflow}@postgres/airflow
    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://${POSTGRES_USER:-airflow}:${POSTGRES_PASSWORD:-airflow}@postgres/airflow
    _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
    IMPORTER_MYSQL_HOST: ${IMPORTER_MYSQL_HOST:-importer-mysql}
    IMPORTER_MYSQL_PORT: ${IMPORTER_MYSQL_PORT:-3306}
    IMPORTER_MYSQL_LOGIN: ${IMPORTER_MYSQL_LOGIN:-importer}
    IMPORTER_MYSQL_PASSWORD: ${IMPORTER_MYSQL_PASSWORD:-importer}
    IMPORTER_MYSQL_SCHEMA: importer
  volumes:
    - ./airflow/opt/airflow/logs:/opt/airflow/logs
    - ${INPUT_DIR:-./importer/var/input}:/var/input
    - ${WORK_DIR:-./importer/var/work}:/var/work
    - ${OUTPUT_DIR:-./importer/var/output}:/var/output
  user: "airflow"
  depends_on:
    &airflow-common-depends-on
    redis:
      condition: service_healthy
    postgres:
      condition: service_healthy
    importer-mysql:
      condition: service_healthy

services:
  postgres:
    hostname: postgres
    image: postgres:13
    environment:
      POSTGRES_USER: ${POSTGRES_USER:-airflow}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-airflow}
      POSTGRES_DB: airflow
    expose:
      - 5432
    volumes:
      - ./postgres/var/lib/postgresql/data/:/var/lib/postgresql/data/
    healthcheck:
      test: [ "CMD", "pg_isready", "-U", "airflow" ]
      interval: 5s
      retries: 5
    restart: unless-stopped

  importer-mysql:
    image: mariadb:10.5.9-focal
    volumes:
      - ./importer-mysql/var/lib/mysql/data:/var/lib/mysql/data
      - ./importer-mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
    expose:
      - 3306
    environment:
      MYSQL_DATABASE: importer
      MYSQL_ROOT_PASSWORD: ${IMPORTER_MYSQL_ROOT_PASSWORD:-importer}
      MYSQL_PASSWORD: ${IMPORTER_MYSQL_PASSWORD:-importer}
      MYSQL_USER: ${IMPORTER_MYSQL_LOGIN:-importer}
    restart: unless-stopped
    healthcheck:
      test:
        [
          "CMD",
          "mysqladmin",
          "ping",
          "--user=$IMPORTER_MYSQL_LOGIN",
          "--password=$IMPORTER_MYSQL_PASSWORD",
          "-h",
          "localhost"
        ]
      interval: 5s
      retries: 5

  redis:
    image: redis:latest
    expose:
      - 6379
    healthcheck:
      test: [ "CMD", "redis-cli", "ping" ]
      interval: 5s
      timeout: 30s
      retries: 50
    restart: unless-stopped

  airflow-webserver:
    <<: *airflow-common
    command: webserver
    healthcheck:
      test:
        [
          "CMD",
          "curl",
          "--fail",
          "http://localhost:8080/health"
        ]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: unless-stopped
    ports:
      - ${HTTP_PORT:-8080}:8080
    environment:
      <<: *airflow-common-env
      AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY:-supersecret}
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-scheduler:
    <<: *airflow-common
    command: scheduler
    healthcheck:
      test:
        [
          "CMD-SHELL",
          'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"'
        ]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: unless-stopped
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-worker:
    <<: *airflow-common
    command: celery worker
    healthcheck:
      test:
        - "CMD-SHELL"
        - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
      interval: 10s
      timeout: 10s
      retries: 5
    environment:
      <<: *airflow-common-env
      # Required to handle warm shutdown of the celery workers properly
      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
      DUMB_INIT_SETSID: "0"
    restart: unless-stopped
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-triggerer:
    <<: *airflow-common
    command: triggerer
    healthcheck:
      test:
        [
          "CMD-SHELL",
          'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'
        ]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: unless-stopped
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-init:
    <<: *airflow-common
    entrypoint: /bin/bash
    # yamllint disable rule:line-length
    command:
      - ./entrypoints/init.sh
    # yamllint enable rule:line-length
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_UPGRADE: 'true'
      _AIRFLOW_WWW_USER_CREATE: 'true'
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
    user: "0:0"
    volumes:
      - ${WORK_DIR:-./importer/var/work}:/var/work
      - ${OUTPUT_DIR:-./importer/var/output}:/var/output

  airflow-cli:
    <<: *airflow-common
    profiles:
      - debug
    environment:
      <<: *airflow-common-env
      CONNECTION_CHECK_MAX_COUNT: "0"
    # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
    command:
      - bash
      - -c
      - airflow

  alembic-cli:
    <<: *airflow-common
    profiles:
      - debug
    volumes:
      - ./importer:/opt/airflow
    entrypoint:
      - alembic
      - -c
      - ./settings/alembic.ini

  flower:
    <<: *airflow-common
    command: celery flower
    healthcheck:
      test: [ "CMD", "curl", "--fail", "http://localhost:5555/" ]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: unless-stopped
    ports:
      - ${FLOWER_PORT:-5555}:5555
    depends_on:
      <<: *airflow-common-depends-on
      airflow-worker:
        condition: service_started
      airflow-init:
        condition: service_completed_successfully