docker-compose.yml

---
version: "3.2"
services:
  zookeeper:
    image: confluentinc/cp-zookeeper
    container_name: zookeeper
    networks:
      - crypto-network
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  kafka:
    # Exposes 9092 for external connections to the broker
    # Use kafka:29092 for connections internal on the docker network
    # See https://rmoff.net/2018/08/02/kafka-listeners-explained/ for details
    image: confluentinc/cp-kafka
    container_name: kafka
    depends_on:
      - zookeeper
    networks:
      - crypto-network
    ports:
      - 9092:9092
      - 30001:30001
    environment:
      KAFKA_CREATE_TOPICS: crypto_raw,crypto_latest_trends,crypto_moving_average
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
      KAFKA_JMX_PORT: 30001
      KAFKA_JMX_HOSTNAME: kafka

  kafka-jmx-exporter:
    build: ./tools/prometheus-jmx-exporter
    container_name: jmx-exporter
    ports:
      - 8080:8080
    links:
      - kafka
    networks:
      - crypto-network
    environment:
      JMX_PORT: 30001
      JMX_HOST: kafka
      HTTP_PORT: 8080
      JMX_EXPORTER_CONFIG_FILE: kafka.yml

  prometheus:
    build: ./tools/prometheus
    container_name: prometheus
    networks:
      - crypto-network
    ports:
      - 9090:9090

  spark-master:
    image: bde2020/spark-master:2.2.2-hadoop2.7
    container_name: spark-master
    networks:
      - crypto-network
    volumes:
      - ./connectors:/connectors
      - ./:/scripts/
    ports:
      - 8082:8080
      - 7077:7077
    environment:
      - INIT_DAEMON_STEP=setup_spark

  spark-worker-1:
    image: bde2020/spark-worker:2.2.2-hadoop2.7
    container_name: spark-worker-1
    networks:
      - crypto-network
    depends_on:
      - spark-master
    ports:
      - 8083:8081
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"

  producer:
    build:
      context: .
      dockerfile: ./Dockerfile.producer
    container_name: producer
    depends_on:
      - kafka
    networks:
      - crypto-network

networks:
  crypto-network:
    driver: "bridge"