docker-compose-spark3.0.2.yml

version: "3"
services:
    namenode:
        image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
        container_name: namenode
        restart: always
        volumes:
            - hadoop_namenode:/hadoop/dfs/name
        environment:
            - CLUSTER_NAME=test
        ports:
            - 9870:9870
            - 9000:9000
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    datanode:
        image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
        container_name: datanode
        restart: always
        depends_on: 
            - namenode
        volumes:
            - hadoop_datanode:/hadoop/dfs/data
        environment:
            SERVICE_PRECONDITION: "namenode:9870"
        ports:
            - 9864:9864
        env_file:
            - ./hadoop.env
        networks:
            - hadoop

    resourcemanager:
        image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
        container_name: resourcemanager
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
        ports:
            - 8088:8088
        env_file:
            - ./hadoop.env
        networks:
            - hadoop
    
    nodemanager:
        image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
        container_name: nodemanager
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
        ports:
            - 8042:8042
        env_file:
            - ./hadoop.env
        networks:
            - hadoop
    
    historyserver:
        image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
        container_name: historyserver
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
        volumes:
            - hadoop_historyserver:/hadoop/yarn/timeline
        ports:
            - 8188:8188
        env_file:
            - ./hadoop.env
        networks:
            - hadoop
    
    hive-server:
        image: bde2020/hive:2.3.2-postgresql-metastore
        container_name: hive-server
        restart: always
        depends_on: 
            - namenode
            - datanode
        environment:
            HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
        ports:
            - 10000:10000
            - 10002:10002
        env_file:
            - ./hadoop.env
        networks:
            - hadoop
    
    hive-metastore:
        image: bde2020/hive:2.3.2-postgresql-metastore
        container_name: hive-metastore
        restart: always
        environment:
            SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432"
        command: /opt/hive/bin/hive --service metastore
        ports:
            - 9083:9083
        env_file:
            - ./hadoop.env
        networks:
            - hadoop
    
    hive-metastore-postgresql:
        image: bde2020/hive-metastore-postgresql:2.3.0
        container_name: hive-metastore-postgresql
        volumes:
            - hive_metastore:/var/lib/postgresql/data
        networks:
            - hadoop
    

        
    spark-master:
        image: bde2020/spark-master:3.0.2-hadoop3.2
        container_name: spark-master
        depends_on:
            - namenode
            - datanode
        environment:
            - INIT_DAEMON_STEP=setup_spark
        ports:
            - 8080:8080
            - 7077:7077
            - 4040:4040
            - 18080:18080
        env_file:
            - ./hadoop.env
        networks:
            - hadoop


    jupyter:
        image: jupyter/pyspark-notebook:latest
        container_name: jupyter
        ports:
        - 8888:8888
        volumes:
        - ./notebooks:/home/jovyan/work  # Mount a local directory to store notebooks
        - ./JDBC_Driver:/Drivers/SQL_Sever/jdbc

        environment:
        - SPARK_MASTER=spark://spark-master:7077  # Point to your Spark master
        networks:
        - hadoop
        

    sql-server:
      image: mcr.microsoft.com/mssql/server:2022-latest
      container_name: sql-server-container
      environment:
        MSSQL_SA_PASSWORD: Mo*012105
        ACCEPT_EULA: Y
      ports:
        - "1433:1433"
      volumes:
        - sql-server-data:/var/opt/mssql
      networks:
        - hadoop



volumes:
    hadoop_namenode:
    hadoop_datanode:
    hadoop_historyserver:
    hive_metastore:
    sql-server-data:
networks:
    hadoop:
        name: hadoop