docker-compose.rancher.yml

version: '2'
services:

  spark-master:
    image: biggis/spark:2.3.2
    hostname: spark-master
    ports:
      - "4040:4040"
      - "6066:6066"
      - "7077:7077"
      - "8080:8080"
    command: start.sh master
    environment:
      USER_ID: 1000
      USER_NAME: spark
      TIMEZONE: Europe/Berlin
      HADOOP_MASTER_ADDRESS: hdfs-name.storage
      REMAINING_SYS_MEM_IN_PERCENT: 20
      SPARK_WORKER_INSTANCS: 2
    labels:
      io.rancher.container.pull_image: always
      io.rancher.scheduler.affinity:host_label: spark-master=true
    volumes:
      - spark-home:/opt/spark
    logging:
      driver: "json-file"
      options:
        max-size: "1m"
        max-file: "1"

  spark-worker:
    image: biggis/spark:2.3.2
    depends_on:
      - spark-master
    command: start.sh worker
    environment:
      USER_ID: 1000
      USER_NAME: spark
      TIMEZONE: Europe/Berlin
      SPARK_MASTER_ADDRESS: spark-master.analytics
      HADOOP_MASTER_ADDRESS: hdfs-name.storage
      REMAINING_SYS_MEM_IN_PERCENT: 20
      SPARK_WORKER_INSTANCS: 2
    labels:
      io.rancher.container.pull_image: always
      io.rancher.scheduler.global: 'true'
    logging:
      driver: "json-file"
      options:
        max-size: "1m"
        max-file: "1"

volumes:
  spark-home:
    driver: rancher-nfs
    external: true