From 4e44ea4bc2043ef7a7b0b9f9db3dce8c6488ea5a Mon Sep 17 00:00:00 2001 From: Sri Harsha Singudasu <105226401+ssingudasu@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:05:32 -0800 Subject: [PATCH] Convert wurstmeister images to bitnami images (#168) * Convert wurstmeister images to bitnami images * Modified ci workflows * Minor changes to README.md * Minor changes to environment variable * ci test271 enable scram * ci changing to SASL_PLAINTEXT * modifying advertized listeners * reverting test271 * modifying test271 authroizer class name * removing listeners * Bumping the Kafka version to 2.7.0 for SCRAM * Minor README.md fixtures * Minor README.md fixtures * Fixtures for kafka versions, KAFKA_IMAGE_TAG * ci fixture for kafka version 0.10.2.1 * Minor README changes * Minor README changes --- .github/workflows/ci.yml | 144 ++++++++++++++++++++++----------------- README.md | 2 +- docker-compose-auth.yml | 70 +++++++++++-------- docker-compose.yml | 117 ++++++++++++++++--------------- 4 files changed, 187 insertions(+), 146 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18183e0b..e1ea588c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,78 +50,86 @@ jobs: services: zookeeper: - image: wurstmeister/zookeeper + image: bitnami/zookeeper:latest ports: - "2181:2181" + env: + ALLOW_ANONYMOUS_LOGIN: yes kafka1: - image: wurstmeister/kafka:2.11-0.10.2.2 + image: bitnami/kafka:0.10.2.1 ports: - "9092:9092" env: KAFKA_BROKER_ID: 1 KAFKA_BROKER_RACK: zone1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + ALLOW_PLAINTEXT_LISTENER: yes KAFKA_ADVERTISED_HOST_NAME: kafka1 KAFKA_ADVERTISED_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 kafka2: - image: wurstmeister/kafka:2.11-0.10.2.2 + image: bitnami/kafka:0.10.2.1 ports: - "9093:9092" env: KAFKA_BROKER_ID: 2 KAFKA_BROKER_RACK: zone1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + ALLOW_PLAINTEXT_LISTENER: yes KAFKA_ADVERTISED_HOST_NAME: kafka2 KAFKA_ADVERTISED_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 kafka3: - image: wurstmeister/kafka:2.11-0.10.2.2 + image: bitnami/kafka:0.10.2.1 ports: - "9094:9092" env: KAFKA_BROKER_ID: 3 KAFKA_BROKER_RACK: zone2 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + ALLOW_PLAINTEXT_LISTENER: yes KAFKA_ADVERTISED_HOST_NAME: kafka3 KAFKA_ADVERTISED_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 kafka4: - image: wurstmeister/kafka:2.11-0.10.2.2 + image: bitnami/kafka:0.10.2.1 ports: - "9095:9092" - env: + env: KAFKA_BROKER_ID: 4 KAFKA_BROKER_RACK: zone2 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + ALLOW_PLAINTEXT_LISTENER: yes KAFKA_ADVERTISED_HOST_NAME: kafka4 KAFKA_ADVERTISED_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 kafka5: - image: wurstmeister/kafka:2.11-0.10.2.2 + image: bitnami/kafka:0.10.2.1 ports: - "9096:9092" - env: + env: KAFKA_BROKER_ID: 5 KAFKA_BROKER_RACK: zone3 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + ALLOW_PLAINTEXT_LISTENER: yes KAFKA_ADVERTISED_HOST_NAME: kafka5 KAFKA_ADVERTISED_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 kafka6: - image: wurstmeister/kafka:2.11-0.10.2.2 + image: bitnami/kafka:0.10.2.1 ports: - "9097:9092" - env: + env: KAFKA_BROKER_ID: 6 KAFKA_BROKER_RACK: zone3 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + ALLOW_PLAINTEXT_LISTENER: yes KAFKA_ADVERTISED_HOST_NAME: kafka6 KAFKA_ADVERTISED_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - test271: + test270: runs-on: ubuntu-latest container: image: cimg/go:1.19 @@ -147,91 +155,99 @@ jobs: services: zookeeper: - image: wurstmeister/zookeeper + image: bitnami/zookeeper:latest ports: - "2181:2181" + env: + ALLOW_ANONYMOUS_LOGIN: yes kafka1: - image: wurstmeister/kafka:2.13-2.7.1 + image: bitnami/kafka:2.7.0 ports: - "9092:9092" env: - KAFKA_BROKER_ID: 1 - KAFKA_BROKER_RACK: zone1 - KAFKA_ADVERTISED_HOST_NAME: kafka1 - KAFKA_ADVERTISED_PORT: 9092 + KAFKA_CFG_BROKER_ID: 1 + KAFKA_CFG_BROKER_RACK: zone1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_ADVERTISED_HOST_NAME: kafka1 + KAFKA_CFG_ADVERTISED_PORT: 9092 + KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer + KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka2: - image: wurstmeister/kafka:2.13-2.7.1 + image: bitnami/kafka:2.7.0 ports: - "9093:9092" env: - KAFKA_BROKER_ID: 2 - KAFKA_BROKER_RACK: zone1 - KAFKA_ADVERTISED_HOST_NAME: kafka2 - KAFKA_ADVERTISED_PORT: 9092 + KAFKA_CFG_BROKER_ID: 2 + KAFKA_CFG_BROKER_RACK: zone1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_ADVERTISED_HOST_NAME: kafka2 + KAFKA_CFG_ADVERTISED_PORT: 9092 + KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer + KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka3: - image: wurstmeister/kafka:2.13-2.7.1 + image: bitnami/kafka:2.7.0 ports: - "9094:9092" env: - KAFKA_BROKER_ID: 3 - KAFKA_BROKER_RACK: zone2 - KAFKA_ADVERTISED_HOST_NAME: kafka3 - KAFKA_ADVERTISED_PORT: 9092 + KAFKA_CFG_BROKER_ID: 3 + KAFKA_CFG_BROKER_RACK: zone2 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_ADVERTISED_HOST_NAME: kafka3 + KAFKA_CFG_ADVERTISED_PORT: 9092 + KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer + KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka4: - image: wurstmeister/kafka:2.13-2.7.1 + image: bitnami/kafka:2.7.0 ports: - "9095:9092" - env: - KAFKA_BROKER_ID: 4 - KAFKA_BROKER_RACK: zone2 - KAFKA_ADVERTISED_HOST_NAME: kafka4 - KAFKA_ADVERTISED_PORT: 9092 + env: + KAFKA_CFG_BROKER_ID: 4 + KAFKA_CFG_BROKER_RACK: zone2 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_ADVERTISED_HOST_NAME: kafka4 + KAFKA_CFG_ADVERTISED_PORT: 9092 + KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer + KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka5: - image: wurstmeister/kafka:2.13-2.7.1 + image: bitnami/kafka:2.7.0 ports: - "9096:9092" - env: - KAFKA_BROKER_ID: 5 - KAFKA_BROKER_RACK: zone3 - KAFKA_ADVERTISED_HOST_NAME: kafka5 - KAFKA_ADVERTISED_PORT: 9092 + env: + KAFKA_CFG_BROKER_ID: 5 + KAFKA_CFG_BROKER_RACK: zone3 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_ADVERTISED_HOST_NAME: kafka5 + KAFKA_CFG_ADVERTISED_PORT: 9092 + KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer + KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka6: - image: wurstmeister/kafka:2.13-2.7.1 + image: bitnami/kafka:2.7.0 ports: - "9097:9092" - env: - KAFKA_BROKER_ID: 6 - KAFKA_BROKER_RACK: zone3 - KAFKA_ADVERTISED_HOST_NAME: kafka6 - KAFKA_ADVERTISED_PORT: 9092 + env: + KAFKA_CFG_BROKER_ID: 6 + KAFKA_CFG_BROKER_RACK: zone3 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_ADVERTISED_HOST_NAME: kafka6 + KAFKA_CFG_ADVERTISED_PORT: 9092 + KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer + KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true snyk: runs-on: ubuntu-latest - needs: [test010, test271] + needs: [test010, test270] steps: - uses: actions/checkout@v3 - name: Run Snyk to check for vulnerabilities diff --git a/README.md b/README.md index 7e77dcc6..a6e460be 100644 --- a/README.md +++ b/README.md @@ -554,7 +554,7 @@ make test You can change the Kafka version of the local cluster by setting the `KAFKA_IMAGE_TAG` environment variable when running `docker-compose up -d`. See the -[`wurstmeister/kafka` dockerhub page](https://hub.docker.com/r/wurstmeister/kafka/tags) for more +[`bitnami/kafka` dockerhub page](https://hub.docker.com/r/bitnami/kafka/tags) for more details on the available versions. #### Run against local cluster diff --git a/docker-compose-auth.yml b/docker-compose-auth.yml index 22d46a46..553e9a0e 100644 --- a/docker-compose-auth.yml +++ b/docker-compose-auth.yml @@ -1,3 +1,8 @@ +# By default, this docker-compose setup uses Kafka 2.7.0. This version can +# be overwritten by setting the KAFKA_IMAGE_TAG environment variable. +# +# See https://hub.docker.com/r/bitnami/kafka/tags for the complete list. +# # This config sets up a simple, single-node cluster that's equipped to use SSL/TLS and/or SASL. # It exposes access on four separate ports: # @@ -7,46 +12,57 @@ # 4. 9095: SASL over SSL # # See examples/auth for the associated cluster configs and certs. -version: '2' +version: '3' services: zookeeper: - image: "wurstmeister/zookeeper:latest" + container_name: zookeeper + hostname: zookeeper + image: bitnami/zookeeper:latest ports: - "2181:2181" + environment: + ALLOW_ANONYMOUS_LOGIN: yes kafka: - image: wurstmeister/kafka:2.13-2.7.1 + container_name: kafka + hostname: kafka + image: bitnami/kafka:${KAFKA_IMAGE_TAG:-2.7.0} + depends_on: + - zookeeper restart: on-failure:3 - links: - - zookeeper ports: - 9092:9092 - 9093:9093 - 9094:9094 - 9095:9095 environment: - KAFKA_BROKER_ID: 1 - KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_ADVERTISED_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_MESSAGE_MAX_BYTES: 200000000 - KAFKA_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094,SASL_SSL://:9095" - KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_PLAINTEXT://localhost:9094,SASL_SSL://localhost:9095" - KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN,SCRAM-SHA-256,SCRAM-SHA-512" - KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer' - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' - KAFKA_SSL_KEYSTORE_LOCATION: /certs/kafka.keystore.jks - KAFKA_SSL_KEYSTORE_PASSWORD: test123 - KAFKA_SSL_KEY_PASSWORD: test123 - KAFKA_SSL_TRUSTSTORE_LOCATION: /certs/kafka.truststore.jks - KAFKA_SSL_TRUSTSTORE_PASSWORD: test123 - KAFKA_SSL_CLIENT_AUTH: none - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf" - CUSTOM_INIT_SCRIPT: |- - echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n };' > /opt/kafka/config/kafka_server_jaas.conf; - /opt/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config 'SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]' --entity-type users --entity-name adminscram + KAFKA_CFG_BROKER_ID: 1 + KAFKA_CFG_BROKER_RACK: zone1 + KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CFG_MESSAGE_MAX_BYTES: 200000000 + KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094,SASL_SSL://:9095" + KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_PLAINTEXT://localhost:9094,SASL_SSL://localhost:9095" + KAFKA_CFG_SASL_ENABLED_MECHANISMS: "PLAIN,SCRAM-SHA-256,SCRAM-SHA-512" + KAFKA_CFG_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" + + KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + + KAFKA_CFG_SSL_KEYSTORE_LOCATION: /opt/bitnami/kafka/config/certs/kafka.truststore.jks + KAFKA_CFG_SSL_KEYSTORE_PASSWORD: test123 + + KAFKA_CFG_SSL_TRUSTSTORE_LOCATION: /opt/bitnami/kafka/config/certs/kafka.truststore.jks + KAFKA_CFG_SSL_TRUSTSTORE_PASSWORD: test123 + + + KAFKA_CFG_SSL_CLIENT_AUTH: none + KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf" + ALLOW_PLAINTEXT_LISTENER: "yes" + entrypoint: + - "/bin/bash" + - "-c" + - echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n };' > /opt/bitnami/kafka/config/kafka_jaas.conf; /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config "SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh volumes: - /var/run/docker.sock:/var/run/docker.sock - - ./examples/auth/certs:/certs + - ./examples/auth/certs:/opt/bitnami/kafka/config/certs diff --git a/docker-compose.yml b/docker-compose.yml index 99a5e5c6..d142a765 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,116 +1,125 @@ -# By default, this docker-compose setup uses Kafka 2.4.1. This version can -# be overwritten by setting the KAFKA_IMAGE_TAG environment variable; some choices here include: +# By default, this docker-compose setup uses Kafka 2.7.0. This version can +# be overwritten by setting the KAFKA_IMAGE_TAG environment variable. # -# 1. Kafka 2.6: 2.13-2.6.0 -# 2. Kafka 2.5: 2.13-2.5.0 -# 3. Kafka 0.10: 2.11-0.10.2.2 -# -# See https://hub.docker.com/r/wurstmeister/kafka/tags for the complete list. -version: '2.1' +# See https://hub.docker.com/r/bitnami/kafka/tags for the complete list. +version: '3' services: zookeeper: - image: wurstmeister/zookeeper + container_name: zookeeper + hostname: zookeeper + image: bitnami/zookeeper:latest ports: - "2181:2181" + environment: + ALLOW_ANONYMOUS_LOGIN: yes # Zone 1 brokers kafka1: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} + container_name: kafka1 + hostname: 169.254.123.123 + image: bitnami/kafka:${KAFKA_IMAGE_TAG:-2.7.0} ports: - "9092:9092" environment: - KAFKA_BROKER_ID: 1 - KAFKA_BROKER_RACK: zone1 - KAFKA_ADVERTISED_HOST_NAME: 169.254.123.123 - KAFKA_ADVERTISED_PORT: 9092 + KAFKA_CFG_BROKER_ID: 1 + KAFKA_CFG_BROKER_RACK: zone1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://169.254.123.123:9092 restart: on-failure depends_on: - zookeeper kafka2: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} + container_name: kafka2 + hostname: 169.254.123.123 + image: bitnami/kafka:${KAFKA_IMAGE_TAG:-2.7.0} ports: - "9093:9092" environment: - KAFKA_BROKER_ID: 2 - KAFKA_BROKER_RACK: zone1 - KAFKA_ADVERTISED_HOST_NAME: 169.254.123.123 - KAFKA_ADVERTISED_PORT: 9093 + KAFKA_CFG_BROKER_ID: 2 + KAFKA_CFG_BROKER_RACK: zone1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://169.254.123.123:9093 restart: on-failure depends_on: - zookeeper - - kafka1 # Zone 2 brokers kafka3: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} + container_name: kafka3 + hostname: 169.254.123.123 + image: bitnami/kafka:${KAFKA_IMAGE_TAG:-2.7.0} ports: - "9094:9092" environment: - KAFKA_BROKER_ID: 3 - KAFKA_BROKER_RACK: zone2 - KAFKA_ADVERTISED_HOST_NAME: 169.254.123.123 - KAFKA_ADVERTISED_PORT: 9094 + KAFKA_CFG_BROKER_ID: 3 + KAFKA_CFG_BROKER_RACK: zone2 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://169.254.123.123:9094 restart: on-failure depends_on: - zookeeper - - kafka1 - - kafka2 kafka4: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} + container_name: kafka4 + hostname: 169.254.123.123 + image: bitnami/kafka:${KAFKA_IMAGE_TAG:-2.7.0} ports: - "9095:9092" environment: - KAFKA_BROKER_ID: 4 - KAFKA_BROKER_RACK: zone2 - KAFKA_ADVERTISED_HOST_NAME: 169.254.123.123 - KAFKA_ADVERTISED_PORT: 9095 + KAFKA_CFG_BROKER_ID: 4 + KAFKA_CFG_BROKER_RACK: zone2 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://169.254.123.123:9095 restart: on-failure depends_on: - zookeeper - - kafka1 - - kafka2 - - kafka3 # Zone 3 brokers kafka5: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} + container_name: kafka5 + hostname: 169.254.123.123 + image: bitnami/kafka:${KAFKA_IMAGE_TAG:-2.7.0} ports: - "9096:9092" environment: - KAFKA_BROKER_ID: 5 - KAFKA_BROKER_RACK: zone3 - KAFKA_ADVERTISED_HOST_NAME: 169.254.123.123 - KAFKA_ADVERTISED_PORT: 9096 + KAFKA_CFG_BROKER_ID: 5 + KAFKA_CFG_BROKER_RACK: zone3 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://169.254.123.123:9096 restart: on-failure depends_on: - zookeeper - - kafka1 - - kafka2 - - kafka3 - - kafka4 kafka6: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} + container_name: kafka6 + hostname: 169.254.123.123 + image: bitnami/kafka:${KAFKA_IMAGE_TAG:-2.7.0} ports: - "9097:9092" environment: - KAFKA_BROKER_ID: 6 - KAFKA_BROKER_RACK: zone3 - KAFKA_ADVERTISED_HOST_NAME: 169.254.123.123 - KAFKA_ADVERTISED_PORT: 9097 + KAFKA_CFG_BROKER_ID: 6 + KAFKA_CFG_BROKER_RACK: zone3 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://169.254.123.123:9097 restart: on-failure depends_on: - zookeeper - - kafka1 - - kafka2 - - kafka3 - - kafka4 - - kafka5