kafka docker-compose.yaml fixes

shotover · Sep 4, 2024 · 1344d23 · 1344d23
1 parent 76b3348
commit 1344d23
Show file tree

Hide file tree

Showing 10 changed files with 30 additions and 22 deletions.
diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-1-rack/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-1-rack/docker-compose.yaml
@@ -24,6 +24,10 @@ services:
       KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "0@kafka0:9093,1@kafka1:9093,2@kafka2:9093"
       KAFKA_CFG_NODE_ID: 0
       ALLOW_PLAINTEXT_LISTENER: "yes"
+      # Required for high availability
+      KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2
     volumes: &volumes
       - type: tmpfs
         target: /bitnami/kafka

diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-2-racks/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-2-racks/docker-compose.yaml
@@ -25,6 +25,10 @@ services:
       KAFKA_CFG_NODE_ID: 0
       KAFKA_CFG_BROKER_RACK: "rack1"
       ALLOW_PLAINTEXT_LISTENER: "yes"
+      # Required for high availability
+      KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2
     volumes: &volumes
       - type: tmpfs
         target: /bitnami/kafka

diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-mtls/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-mtls/docker-compose.yaml
@@ -27,6 +27,10 @@ services:
       KAFKA_TLS_CLIENT_AUTH: required
       KAFKA_CFG_AUTHORIZER_CLASS_NAME: "org.apache.kafka.metadata.authorizer.StandardAuthorizer"
       KAFKA_CFG_SUPER_USERS: "User:O=ShotoverTestCertificate,CN=Generic-Cert"
+      # Required for high availability
+      KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2
     volumes: &volumes
       - type: tmpfs
         target: /bitnami/kafka

diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-sasl-plain/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-sasl-plain/docker-compose.yaml
@@ -25,14 +25,14 @@ services:
       KAFKA_CLIENT_PASSWORDS: "password"
       KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
       KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL: "PLAIN"
-      KAFKA_CONTROLLER_USER: "controller_user"
-      KAFKA_CONTROLLER_PASSWORD: "controller_password"
       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "BROKER"
       KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: "PLAIN"
-      KAFKA_INTER_BROKER_USER: "controller_user"
-      KAFKA_INTER_BROKER_PASSWORD: "controller_password"
       KAFKA_CERTIFICATE_PASSWORD: "123456"
       KAFKA_KRAFT_CLUSTER_ID: "abcdefghijklmnopqrstuv"
+      # Required for high availability
+      KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2
     volumes: &volumes
       - type: tmpfs
         target: /bitnami/kafka

diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram-over-mtls/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram-over-mtls/docker-compose.yaml
@@ -28,8 +28,8 @@ services:
       KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL: "PLAIN"
       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "BROKER"
       KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: "SCRAM-SHA-256"
-      KAFKA_INTER_BROKER_USER: "controller_user"
-      KAFKA_INTER_BROKER_PASSWORD: "controller_password"
+      KAFKA_INTER_BROKER_USER: "super_user"
+      KAFKA_INTER_BROKER_PASSWORD: "super_password"
       KAFKA_CERTIFICATE_PASSWORD: "password"
       KAFKA_KRAFT_CLUSTER_ID: "abcdefghijklmnopqrstuv"
       KAFKA_TLS_CLIENT_AUTH: required
@@ -38,6 +38,10 @@ services:
       # * the user named `user`
       # * any clients connected via a TLS certificate of `O=ShotoverTestCertificate,CN=Generic-Cert`
       KAFKA_CFG_SUPER_USERS: "User:super_user;User:O=ShotoverTestCertificate,CN=Generic-Cert"
+      # Required for high availability
+      KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2
     volumes: &volumes
       - type: tmpfs
         target: /bitnami/kafka

diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram/docker-compose.yaml
@@ -25,14 +25,14 @@ services:
       KAFKA_CLIENT_PASSWORDS: "password"
       KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
       KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL: "PLAIN"
-      KAFKA_CONTROLLER_USER: "controller_user"
-      KAFKA_CONTROLLER_PASSWORD: "controller_password"
       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "BROKER"
       KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: "SCRAM-SHA-256"
-      KAFKA_INTER_BROKER_USER: "controller_user"
-      KAFKA_INTER_BROKER_PASSWORD: "controller_password"
       KAFKA_CERTIFICATE_PASSWORD: "123456"
       KAFKA_KRAFT_CLUSTER_ID: "abcdefghijklmnopqrstuv"
+      # Required for high availability
+      KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2
     volumes: &volumes
       - type: tmpfs
         target: /bitnami/kafka

diff --git a/shotover-proxy/tests/test-configs/kafka/cluster-tls/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/cluster-tls/docker-compose.yaml
@@ -25,6 +25,10 @@ services:
       KAFKA_CFG_NODE_ID: 0
       KAFKA_CERTIFICATE_PASSWORD: password
       KAFKA_TLS_CLIENT_AUTH: none
+      # Required for high availability
+      KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
+      KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: 2
     volumes: &volumes
       - type: tmpfs
         target: /bitnami/kafka

diff --git a/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-plain/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-plain/docker-compose.yaml
@@ -15,12 +15,8 @@ services:
       - KAFKA_CLIENT_PASSWORDS=password
       - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
       - KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL=PLAIN
-      - KAFKA_CONTROLLER_USER=controller_user
-      - KAFKA_CONTROLLER_PASSWORD=controller_password
       - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=BROKER
       - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN
-      - KAFKA_INTER_BROKER_USER=controller_user
-      - KAFKA_INTER_BROKER_PASSWORD=controller_password
     volumes:
       - type: tmpfs
         target: /bitnami/kafka
diff --git a/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-scram/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka/passthrough-sasl-scram/docker-compose.yaml
@@ -13,13 +13,9 @@ services:
       - KAFKA_CLIENT_USERS=user
       - KAFKA_CLIENT_PASSWORDS=password
       - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
-      - KAFKA_CONTROLLER_USER=controller_user
-      - KAFKA_CONTROLLER_PASSWORD=controller_password
       - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=BROKER
       - KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL=PLAIN
       - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=SCRAM-SHA-256
-      - KAFKA_INTER_BROKER_USER=controller_user
-      - KAFKA_INTER_BROKER_PASSWORD=controller_password
     volumes:
       - type: tmpfs
         target: /bitnami/kafka
diff --git a/.../tests/test-configs/kafka/single-sasl-scram-plaintext-source-tls-sink/docker-compose.yaml b/.../tests/test-configs/kafka/single-sasl-scram-plaintext-source-tls-sink/docker-compose.yaml
@@ -13,13 +13,9 @@ services:
       - KAFKA_CLIENT_USERS=user
       - KAFKA_CLIENT_PASSWORDS=password
       - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
-      - KAFKA_CONTROLLER_USER=controller_user
-      - KAFKA_CONTROLLER_PASSWORD=controller_password
       - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=BROKER
       - KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL=PLAIN
       - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=SCRAM-SHA-256
-      - KAFKA_INTER_BROKER_USER=controller_user
-      - KAFKA_INTER_BROKER_PASSWORD=controller_password
       - KAFKA_CERTIFICATE_PASSWORD=password
       - KAFKA_TLS_CLIENT_AUTH=none
     volumes: