From 1344d235d28f325b48d34cfe574c4555de5575cb Mon Sep 17 00:00:00 2001 From: Lucas Kent Date: Wed, 4 Sep 2024 15:46:46 +1000 Subject: [PATCH] kafka docker-compose.yaml fixes --- .../test-configs/kafka/cluster-1-rack/docker-compose.yaml | 4 ++++ .../kafka/cluster-2-racks/docker-compose.yaml | 4 ++++ .../test-configs/kafka/cluster-mtls/docker-compose.yaml | 4 ++++ .../kafka/cluster-sasl-plain/docker-compose.yaml | 8 ++++---- .../cluster-sasl-scram-over-mtls/docker-compose.yaml | 8 ++++++-- .../kafka/cluster-sasl-scram/docker-compose.yaml | 8 ++++---- .../test-configs/kafka/cluster-tls/docker-compose.yaml | 4 ++++ .../kafka/passthrough-sasl-plain/docker-compose.yaml | 4 ---- .../kafka/passthrough-sasl-scram/docker-compose.yaml | 4 ---- .../docker-compose.yaml | 4 ---- 10 files changed, 30 insertions(+), 22 deletions(-) diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-1-rack/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-1-rack/docker-compose.yaml index 5dfc8c51a..d1238b36e 100644 --- a/shotover-proxy/tests/test-configs/kafka/cluster-1-rack/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/cluster-1-rack/docker-compose.yaml @@ -24,6 +24,10 @@ services: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "0@kafka0:9093,1@kafka1:9093,2@kafka2:9093" KAFKA_CFG_NODE_ID: 0 ALLOW_PLAINTEXT_LISTENER: "yes" + # Required for high availability + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2 volumes: &volumes - type: tmpfs target: /bitnami/kafka diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-2-racks/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-2-racks/docker-compose.yaml index 5842425d6..e1a5b226c 100644 --- a/shotover-proxy/tests/test-configs/kafka/cluster-2-racks/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/cluster-2-racks/docker-compose.yaml @@ -25,6 +25,10 @@ services: KAFKA_CFG_NODE_ID: 0 KAFKA_CFG_BROKER_RACK: "rack1" ALLOW_PLAINTEXT_LISTENER: "yes" + # Required for high availability + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2 volumes: &volumes - type: tmpfs target: /bitnami/kafka diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-mtls/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-mtls/docker-compose.yaml index d78889598..dea55ff05 100644 --- a/shotover-proxy/tests/test-configs/kafka/cluster-mtls/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/cluster-mtls/docker-compose.yaml @@ -27,6 +27,10 @@ services: KAFKA_TLS_CLIENT_AUTH: required KAFKA_CFG_AUTHORIZER_CLASS_NAME: "org.apache.kafka.metadata.authorizer.StandardAuthorizer" KAFKA_CFG_SUPER_USERS: "User:O=ShotoverTestCertificate,CN=Generic-Cert" + # Required for high availability + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2 volumes: &volumes - type: tmpfs target: /bitnami/kafka diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-sasl-plain/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-sasl-plain/docker-compose.yaml index aa8718808..2acf245a5 100644 --- a/shotover-proxy/tests/test-configs/kafka/cluster-sasl-plain/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/cluster-sasl-plain/docker-compose.yaml @@ -25,14 +25,14 @@ services: KAFKA_CLIENT_PASSWORDS: "password" KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER" KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL: "PLAIN" - KAFKA_CONTROLLER_USER: "controller_user" - KAFKA_CONTROLLER_PASSWORD: "controller_password" KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "BROKER" KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: "PLAIN" - KAFKA_INTER_BROKER_USER: "controller_user" - KAFKA_INTER_BROKER_PASSWORD: "controller_password" KAFKA_CERTIFICATE_PASSWORD: "123456" KAFKA_KRAFT_CLUSTER_ID: "abcdefghijklmnopqrstuv" + # Required for high availability + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2 volumes: &volumes - type: tmpfs target: /bitnami/kafka diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram-over-mtls/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram-over-mtls/docker-compose.yaml index 83567db4b..89181febd 100644 --- a/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram-over-mtls/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram-over-mtls/docker-compose.yaml @@ -28,8 +28,8 @@ services: KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL: "PLAIN" KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "BROKER" KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: "SCRAM-SHA-256" - KAFKA_INTER_BROKER_USER: "controller_user" - KAFKA_INTER_BROKER_PASSWORD: "controller_password" + KAFKA_INTER_BROKER_USER: "super_user" + KAFKA_INTER_BROKER_PASSWORD: "super_password" KAFKA_CERTIFICATE_PASSWORD: "password" KAFKA_KRAFT_CLUSTER_ID: "abcdefghijklmnopqrstuv" KAFKA_TLS_CLIENT_AUTH: required @@ -38,6 +38,10 @@ services: # * the user named `user` # * any clients connected via a TLS certificate of `O=ShotoverTestCertificate,CN=Generic-Cert` KAFKA_CFG_SUPER_USERS: "User:super_user;User:O=ShotoverTestCertificate,CN=Generic-Cert" + # Required for high availability + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2 volumes: &volumes - type: tmpfs target: /bitnami/kafka diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram/docker-compose.yaml index da04178e5..792f913f2 100644 --- a/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram/docker-compose.yaml @@ -25,14 +25,14 @@ services: KAFKA_CLIENT_PASSWORDS: "password" KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER" KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL: "PLAIN" - KAFKA_CONTROLLER_USER: "controller_user" - KAFKA_CONTROLLER_PASSWORD: "controller_password" KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "BROKER" KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: "SCRAM-SHA-256" - KAFKA_INTER_BROKER_USER: "controller_user" - KAFKA_INTER_BROKER_PASSWORD: "controller_password" KAFKA_CERTIFICATE_PASSWORD: "123456" KAFKA_KRAFT_CLUSTER_ID: "abcdefghijklmnopqrstuv" + # Required for high availability + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2 volumes: &volumes - type: tmpfs target: /bitnami/kafka diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-tls/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-tls/docker-compose.yaml index 6d7521c51..7c2724660 100644 --- a/shotover-proxy/tests/test-configs/kafka/cluster-tls/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/cluster-tls/docker-compose.yaml @@ -25,6 +25,10 @@ services: KAFKA_CFG_NODE_ID: 0 KAFKA_CERTIFICATE_PASSWORD: password KAFKA_TLS_CLIENT_AUTH: none + # Required for high availability + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 + KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2 volumes: &volumes - type: tmpfs target: /bitnami/kafka diff --git a/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-plain/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-plain/docker-compose.yaml index db35a577f..eaa424da5 100644 --- a/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-plain/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-plain/docker-compose.yaml @@ -15,12 +15,8 @@ services: - KAFKA_CLIENT_PASSWORDS=password - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER - KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL=PLAIN - - KAFKA_CONTROLLER_USER=controller_user - - KAFKA_CONTROLLER_PASSWORD=controller_password - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=BROKER - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN - - KAFKA_INTER_BROKER_USER=controller_user - - KAFKA_INTER_BROKER_PASSWORD=controller_password volumes: - type: tmpfs target: /bitnami/kafka diff --git a/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-scram/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-scram/docker-compose.yaml index eb3338f57..02b9686f5 100644 --- a/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-scram/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-scram/docker-compose.yaml @@ -13,13 +13,9 @@ services: - KAFKA_CLIENT_USERS=user - KAFKA_CLIENT_PASSWORDS=password - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER - - KAFKA_CONTROLLER_USER=controller_user - - KAFKA_CONTROLLER_PASSWORD=controller_password - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=BROKER - KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL=PLAIN - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=SCRAM-SHA-256 - - KAFKA_INTER_BROKER_USER=controller_user - - KAFKA_INTER_BROKER_PASSWORD=controller_password volumes: - type: tmpfs target: /bitnami/kafka diff --git a/shotover-proxy/tests/test-configs/kafka/single-sasl-scram-plaintext-source-tls-sink/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/single-sasl-scram-plaintext-source-tls-sink/docker-compose.yaml index 79b3522f6..e887a8921 100644 --- a/shotover-proxy/tests/test-configs/kafka/single-sasl-scram-plaintext-source-tls-sink/docker-compose.yaml +++ b/shotover-proxy/tests/test-configs/kafka/single-sasl-scram-plaintext-source-tls-sink/docker-compose.yaml @@ -13,13 +13,9 @@ services: - KAFKA_CLIENT_USERS=user - KAFKA_CLIENT_PASSWORDS=password - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER - - KAFKA_CONTROLLER_USER=controller_user - - KAFKA_CONTROLLER_PASSWORD=controller_password - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=BROKER - KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL=PLAIN - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=SCRAM-SHA-256 - - KAFKA_INTER_BROKER_USER=controller_user - - KAFKA_INTER_BROKER_PASSWORD=controller_password - KAFKA_CERTIFICATE_PASSWORD=password - KAFKA_TLS_CLIENT_AUTH=none volumes: