docker-compose.yml

version: "3.6"
services:
  # Master
  master-node:
    image: "jwaresolutions/big-data-cluster:0.4.1"
    container_name: "master-node"
    restart: "always"
    command: bash -c "/home/big_data/spark-cmd.sh start master-node"
    ports:
      - 8088:8088
      - 8080:8080
      - 9870:9870
      - 18080:18080
    networks:
      - cluster-net
    volumes:
      # - "./data:/home/big_data/data" # Your data
      - hdfs-master-data:/home/hadoop/data/nameNode
      - hdfs-master-checkpoint-data:/home/hadoop/data/namesecondary

  # Workers
  worker:
    image: "jwaresolutions/big-data-cluster:0.4.1"
    restart: "always"
    command: bash -c "/home/big_data/spark-cmd.sh start"
    deploy:
    depends_on:
      - "master-node"
    volumes:
      - hdfs-worker-data:/home/hadoop/data/dataNode
    networks:
      - cluster-net

volumes:
  hdfs-master-data:
  hdfs-master-checkpoint-data:
  hdfs-worker-data:

# Create the cluster-net network
networks:
  cluster-net:
    name: "cluster_net" # Useful for format as it does not allow '-' char on command
    driver: bridge
    attachable: false # Attachable: true prevents user to connect to Hadoop panels