docker-compose-hive.yml

version: '3' 
services:

  namenode:
    image: bde2020/hadoop-namenode:1.1.0-hadoop2.8-java8
    container_name: namenode
    volumes:
      - ${HOME}/data-docker/docker-spark-bde2020-zeppelin/data/namenode:/hadoop/dfs/name:rw
    environment:
      - CLUSTER_NAME=test
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    healthcheck:
      interval: 5s
      retries: 10
    env_file:
      - ./hadoop-hive.env
    ports:
      - 50070:50070
    networks:
      - spark-net
      
  datanode:
    image: bde2020/hadoop-datanode:1.1.0-hadoop2.8-java8
    container_name: datanode
    depends_on: 
      - namenode
    volumes:
      - ${HOME}/data-docker/docker-spark-bde2020-zeppelin/data/datanode:/hadoop/dfs/data:rw
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    healthcheck:
      interval: 5s
      retries: 10
    env_file:
      - ./hadoop-hive.env
    ports:
      - 50075:50075
    networks:
      - spark-net

  hive-server:
    image: bde2020/hive:2.1.0-postgresql-metastore
    container_name: hive-server
    env_file:
      - ./hadoop-hive.env
    environment:
      - HIVE_CORE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore/metastore
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    depends_on:
      - namenode
    healthcheck:
      interval: 5s
      retries: 10
    ports:
      - "10000:10000"
    networks:
      - spark-net
      
  hive-metastore:
    image: bde2020/hive:2.1.0-postgresql-metastore
    container_name: hive-metastore
    env_file:
      - ./hadoop-hive.env
    environment:
      - HIVE_CORE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore/metastore
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    command: /opt/hive/bin/hive --service metastore
    networks:
      - spark-net
    
  hive-metastore-postgresql:
    image: bde2020/hive-metastore-postgresql:2.1.0
    container_name: hive-metastore-postgresql
    environment:
      - HIVE_CORE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore/metastore
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    networks:
      - spark-net
    
  spark-master:
    image: bde2020/spark-master:2.1.0-hadoop2.8-hive-java8
    container_name: spark-master
    ports:
      - 8080:8080
      - 7077:7077
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    depends_on:
      - namenode
      - datanode
    healthcheck:
      interval: 5s
      retries: 10 
    env_file:
      - ./hadoop-hive.env
    networks:
      - spark-net
     
  spark-worker:
    image: bde2020/spark-worker:2.1.0-hadoop2.8-hive-java8
    container_name: spark-worker
    depends_on:
      - spark-master
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
      - HIVE_CORE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore/metastore
    healthcheck:
      interval: 5s
      retries: 12 
    ports:
      - 8081:8081
    env_file:
      - ./hadoop-hive.env
    networks:
      - spark-net

  zeppelin:
    image: openkbs/docker-spark-bde2020-zeppelin
    #init: true
    build:
      context: .
      args:
        APACHE_SPARK_VERSION: 2.1.0 
        APACHE_HADOOP_VERSION: 2.8.0 
        SPARK_MASTER: "spark://spark-master:7077" 
        ZEPPELIN_DOWNLOAD_URL: http://apache.cs.utah.edu/zeppelin 
        ZEPPELIN_VERSION: 0.7.3 
        ZEPPELIN_PORT: 8080 
        ZEPPELIN_INSTALL_DIR: /usr/lib 
        ZEPPELIN_HOME: /usr/lib/zeppelin 
        ZEPPELIN_PKG_NAME: "zeppelin-${ZEPPELIN_VERSION:-0.7.3}-bin-all"
    container_name: zeppelin
    environment:
      CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
      SPARK_MASTER: "spark://spark-master:7077"
      MASTER: "spark://spark-master:7077"
      ZEPPELIN_PORT: 8080
      ZEPPELIN_JAVA_OPTS: >-
        -Dspark.driver.memory=6g
        -Dspark.executor.memory=24g
        -Dspark.cores.max=16
      ZEPPELIN_MEM: "-Xms8g -Xmx30g"
      ZEPPELIN_INTP_MEM: "-Xms6g -Xmx30g -XX:MaxPermSize=4096m"
      #SPARK_SUBMIT_OPTIONS: "--jars /opt/sansa-examples/jars/sansa-examples-spark-2016-12.jar"
# export ZEPPELIN_JAVA_OPTS      		# Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
# export ZEPPELIN_MEM            		# Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
# export ZEPPELIN_INTP_MEM       		# zeppelin interpreter process jvm mem options. Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
# export ZEPPELIN_INTP_JAVA_OPTS 		# zeppelin interpreter process jvm options.
    ports:
      - 19090:8080
    env_file:
      - ./hadoop-hive.env
    volumes:
      - ${HOME}/data-docker/docker-spark-bde2020-zeppelin/data:/usr/lib/zeppelin/data:rw
      - ${HOME}/data-docker/docker-spark-bde2020-zeppelin/notebook:/usr/lib/zeppelin/notebook:rw
    depends_on:
      - spark-master
      - namenode
    networks:
      - spark-net
    
  spark-notebook:
    image: bde2020/spark-notebook:2.1.0-hadoop2.8-hive
    container_name: spark-notebook
    env_file:
      - ./hadoop-hive.env
    ports:
      - 9001:9001
    depends_on:
      - spark-master
    networks:
      - spark-net 

  hue:
    image: bde2020/hdfs-filebrowser:3.11
    container_name: hue
    ports:
      - 8088:8088
    environment:
      - NAMENODE_HOST=namenode
    depends_on:
      - namenode
    networks:
      - spark-net

networks:
  spark-net:
    external:
      name: spark-net