docker-compose.yml

# https://docs.confluent.io/current/installation/docker/config-reference.html
# https://github.com/confluentinc/cp-docker-images

version: "3"

services:

  zookeeper:
    container_name: local-zookeeper
    image: confluentinc/cp-zookeeper:5.5.1
    ports:
      - 2181:2181
    hostname: zookeeper
    networks:
      - local_kafka_network
    environment:
      - ZOOKEEPER_CLIENT_PORT=2181

  kafka:
    container_name: local-kafka
    image: confluentinc/cp-kafka:5.5.1
    depends_on:
      - zookeeper
    ports:
      - 9092:9092
      - 29092:29092
    hostname: kafka
    networks:
      - local_kafka_network
    environment:
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1

  schema-registry:
    container_name: local-schema-registry
    image: confluentinc/cp-schema-registry:5.5.1
    depends_on:
      - kafka
    ports:
      - 8081:8081
    hostname: schema-registry
    networks:
      - local_kafka_network
    environment:
      - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
      - SCHEMA_REGISTRY_HOST_NAME=schema-registry
      - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081
      - SCHEMA_REGISTRY_DEBUG=true
    command:
      - /bin/bash
      - -c
      - |
        # install jq
        curl -sL https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /usr/local/bin/jq && chmod u+x /usr/local/bin/jq
        # start
        /etc/confluent/docker/run

  schema-registry-ui:
    container_name: local-schema-registry-ui
    image: landoop/schema-registry-ui:latest
    depends_on:
      - schema-registry
    ports:
      - 8001:8000
    hostname: schema-registry-ui
    networks:
      - local_kafka_network
    environment:
      - SCHEMAREGISTRY_URL=http://schema-registry:8081
      - PROXY=true

  kafka-rest:
    container_name: local-kafka-rest
    image: confluentinc/cp-kafka-rest:5.5.1
    depends_on:
      - kafka
      - schema-registry
    ports:
      - 8082:8082
    hostname: kafka-rest
    networks:
      - local_kafka_network
    environment:
      - KAFKA_REST_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_REST_LISTENERS=http://kafka-rest:8082
      - KAFKA_REST_SCHEMA_REGISTRY_URL=http://schema-registry:8081
      - KAFKA_REST_HOST_NAME=kafka-rest

  kafka-ui:
    container_name: local-kafka-ui
    image: landoop/kafka-topics-ui:latest
    depends_on:
      - kafka-rest
    ports:
      - 8000:8000
    hostname: kafka-ui
    networks:
      - local_kafka_network
    environment:
      - KAFKA_REST_PROXY_URL=http://kafka-rest:8082
      - PROXY=true

  # https://github.com/confluentinc/ksql/blob/4.1.3-post/docs/tutorials/docker-compose.yml#L85
  ksql-server:
    container_name: local-ksql-server
    # TODO update 5.5.1
    image: confluentinc/cp-ksql-server:5.4.2
    depends_on:
      - kafka
      - schema-registry
    ports:
      - 8088:8088
    hostname: ksql-server
    networks:
      - local_kafka_network
    environment:
      - KSQL_BOOTSTRAP_SERVERS=kafka:29092
      - KSQL_LISTENERS=http://ksql-server:8088
      - KSQL_KSQL_SCHEMA_REGISTRY_URL=http://schema-registry:8081
      - KSQL_KSQL_SERVICE_ID=local-ksql-server

  ksql-cli:
    container_name: local-ksql-cli
    # TODO update 5.5.1
    image: confluentinc/cp-ksql-cli:5.4.2
    depends_on:
      - ksql-server
    hostname: ksql-cli
    networks:
      - local_kafka_network
    entrypoint: /bin/sh
    tty: true

  # distributed mode
  kafka-connect:
    container_name: local-kafka-connect
    image: confluentinc/cp-kafka-connect:5.5.1
    depends_on:
      - kafka
      - schema-registry
    ports:
      - 8083:8083
    hostname: kafka-connect
    networks:
      - local_kafka_network
    environment:
      - CONNECT_BOOTSTRAP_SERVERS=kafka:29092
      - CONNECT_REST_ADVERTISED_HOST_NAME=kafka-connect
      - CONNECT_REST_PORT=8083
      - CONNECT_GROUP_ID=local-connect-group
      - CONNECT_CONFIG_STORAGE_TOPIC=local-connect-configs
      - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR=1
      - CONNECT_OFFSET_FLUSH_INTERVAL_MS=10000
      - CONNECT_OFFSET_STORAGE_TOPIC=local-connect-offsets
      - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR=1
      - CONNECT_STATUS_STORAGE_TOPIC=local-connect-status
      - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR=1
      - CONNECT_KEY_CONVERTER=io.confluent.connect.avro.AvroConverter
      - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL=http://schema-registry:8081
      - CONNECT_VALUE_CONVERTER=io.confluent.connect.avro.AvroConverter
      - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL=http://schema-registry:8081
      - CONNECT_INTERNAL_KEY_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_INTERNAL_VALUE_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_PLUGIN_PATH=/usr/share/java
    volumes:
      - "./local/connect/data:/data"
    command:
      - /bin/bash
      - -c
      - |
        # install unzip
        apt-get update && apt-get install unzip -y
        # install plugin
        unzip /data/jcustenborder-kafka-connect-spooldir-*.zip 'jcustenborder-kafka-connect-spooldir-*/lib/*' -d /usr/share/java/kafka-connect-spooldir/
        mv /usr/share/java/kafka-connect-spooldir/*/lib/* /usr/share/java/kafka-connect-spooldir
        ls -la /usr/share/java
        # setup spooldir plugin
        mkdir -p /tmp/error /tmp/finished
        # start
        /etc/confluent/docker/run

  kafka-connect-ui:
    container_name: local-kafka-connect-ui
    image: landoop/kafka-connect-ui:latest
    depends_on:
      - kafka-connect
    ports:
      - 8002:8000
    hostname: kafka-connect-ui
    networks:
      - local_kafka_network
    environment:
      - CONNECT_URL=http://kafka-connect:8083

networks:
  local_kafka_network: