Split ShotoverNodeConfig::address into address_for_peers and address_…

…for_client (#1780)

changelog.md

@@ -13,6 +13,7 @@ This assists us in knowing when to make the next release a breaking release and
 ### topology.yaml
 
 * A new mandatory configuration `check_shotover_peers_delay_ms` is added for `KafkaSinkCluster`. See [transform.md](docs/src/transforms.md) for details on this configuration.
+* The `address` field for each shotover node in `KafkaSinkCluster` is replaced with `address_for_clients` and `address_for_peers`. See [transform.md](docs/src/transforms.md) for details on these fields.
 
 ## 0.4.0
 

docs/src/transforms.md

@@ -261,10 +261,14 @@ If SCRAM authentication against the first kafka broker fails, shotover will term
     # A list of every Shotover node that will be proxying to the same kafka cluster.
     # This field should be identical for all Shotover nodes proxying to the same kafka cluster.
     shotover_nodes:
-        # Address of the Shotover node.
+        # Address of the Shotover node that is reported to the kafka clients.
         # This is usually the same address as the Shotover source that is connected to this sink.
-        # But it may be different if you want Shotover to report a different address.
-      - address: "127.0.0.1:9092"
+        # But it may be different if you want Shotover to report a different address to its clients.
+      - address_for_client: "127.0.0.1:9092"
+        # Address of the shotover node as used to check for peers that are up.
+        # This is usually the same address as the Shotover source that is connected to this sink.
+        # But it may be different if you want Shotover to connect to its peers via a different address.
+        address_for_peers: "127.0.0.1:9092"
         # The rack the Shotover node will report as and route messages to.
         # For performance reasons, the Shotover node should be physically located in this rack.
         rack: "rack0"

shotover-proxy/benches/windsock/kafka/bench.rs

@@ -100,7 +100,8 @@ impl KafkaBench {
                 check_shotover_peers_delay_ms: Some(3000),
                 first_contact_points: vec![kafka_address],
                 shotover_nodes: vec![ShotoverNodeConfig {
-                    address: host_address.parse().unwrap(),
+                    address_for_clients: host_address.parse().unwrap(),
+                    address_for_peers: host_address.parse().unwrap(),
                     rack: "rack1".into(),
                     broker_id: 0,
                 }],

shotover-proxy/tests/test-configs/kafka/cluster-1-rack/topology-single.yaml

@@ -6,7 +6,8 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 0
             local_shotover_broker_id: 0

shotover-proxy/tests/test-configs/kafka/cluster-1-rack/topology1.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9191"
+              - address_for_peers: "127.0.0.1:9191"
+                address_for_clients: "127.0.0.1:9191"
                 rack: "rack0"
                 broker_id: 0
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 1
-              - address: "127.0.0.1:9193"
+              - address_for_peers: "127.0.0.1:9193"
+                address_for_clients: "127.0.0.1:9193"
                 rack: "rack0"
                 broker_id: 2
             local_shotover_broker_id: 0

shotover-proxy/tests/test-configs/kafka/cluster-1-rack/topology2.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9191"
+              - address_for_peers: "127.0.0.1:9191"
+                address_for_clients: "127.0.0.1:9191"
                 rack: "rack0"
                 broker_id: 0
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 1
-              - address: "127.0.0.1:9193"
+              - address_for_peers: "127.0.0.1:9193"
+                address_for_clients: "127.0.0.1:9193"
                 rack: "rack0"
                 broker_id: 2
             local_shotover_broker_id: 1

shotover-proxy/tests/test-configs/kafka/cluster-1-rack/topology3.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9191"
+              - address_for_peers: "127.0.0.1:9191"
+                address_for_clients: "127.0.0.1:9191"
                 rack: "rack0"
                 broker_id: 0
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 1
-              - address: "127.0.0.1:9193"
+              - address_for_peers: "127.0.0.1:9193"
+                address_for_clients: "127.0.0.1:9193"
                 rack: "rack0"
                 broker_id: 2
             local_shotover_broker_id: 2

shotover-proxy/tests/test-configs/kafka/cluster-2-racks/topology-rack1.yaml

@@ -6,10 +6,12 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "localhost:9191"
+              - address_for_peers: "localhost:9191"
+                address_for_clients: "127.0.0.1:9191"
                 rack: "rack1"
                 broker_id: 0
-              - address: "localhost:9192"
+              - address_for_peers: "localhost:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack2"
                 broker_id: 1
             local_shotover_broker_id: 0

shotover-proxy/tests/test-configs/kafka/cluster-2-racks/topology-rack2.yaml

@@ -6,10 +6,12 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "localhost:9191"
+              - address_for_peers: "localhost:9191"
+                address_for_clients: "127.0.0.1:9191"
                 rack: "rack1"
                 broker_id: 0
-              - address: "localhost:9192"
+              - address_for_peers: "localhost:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack2"
                 broker_id: 1
             local_shotover_broker_id: 1

shotover-proxy/tests/test-configs/kafka/cluster-3-racks/topology-rack1.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "localhost:9191"
+              - address_for_peers: "localhost:9191"
+                address_for_clients: "localhost:9191"
                 rack: "rack1"
                 broker_id: 0
-              - address: "localhost:9192"
+              - address_for_peers: "localhost:9192"
+                address_for_clients: "localhost:9192"
                 rack: "rack2"
                 broker_id: 1
-              - address: "localhost:9193"
+              - address_for_peers: "localhost:9193"
+                address_for_clients: "localhost:9193"
                 rack: "rack3"
                 broker_id: 2
             local_shotover_broker_id: 0

shotover-proxy/tests/test-configs/kafka/cluster-3-racks/topology-rack2.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "localhost:9191"
+              - address_for_peers: "localhost:9191"
+                address_for_clients: "localhost:9191"
                 rack: "rack1"
                 broker_id: 0
-              - address: "localhost:9192"
+              - address_for_peers: "localhost:9192"
+                address_for_clients: "localhost:9192"
                 rack: "rack2"
                 broker_id: 1
-              - address: "localhost:9193"
+              - address_for_peers: "localhost:9193"
+                address_for_clients: "localhost:9193"
                 rack: "rack3"
                 broker_id: 2
             local_shotover_broker_id: 1

shotover-proxy/tests/test-configs/kafka/cluster-3-racks/topology-rack3.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "localhost:9191"
+              - address_for_peers: "localhost:9191"
+                address_for_clients: "localhost:9191"
                 rack: "rack1"
                 broker_id: 0
-              - address: "localhost:9192"
+              - address_for_peers: "localhost:9192"
+                address_for_clients: "localhost:9192"
                 rack: "rack2"
                 broker_id: 1
-              - address: "localhost:9193"
+              - address_for_peers: "localhost:9193"
+                address_for_clients: "localhost:9193"
                 rack: "rack3"
                 broker_id: 2
             local_shotover_broker_id: 2

shotover-proxy/tests/test-configs/kafka/cluster-mtls/topology.yaml

@@ -9,7 +9,8 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "localhost:9192"
+                address_for_clients: "localhost:9192"
                 rack: "rack0"
                 broker_id: 0
             local_shotover_broker_id: 0

shotover-proxy/tests/test-configs/kafka/cluster-sasl-plain/topology-single.yaml

@@ -6,7 +6,8 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 0
             local_shotover_broker_id: 0

shotover-proxy/tests/test-configs/kafka/cluster-sasl-plain/topology1.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9191"
+              - address_for_peers: "127.0.0.1:9191"
+                address_for_clients: "127.0.0.1:9191"
                 rack: "rack0"
                 broker_id: 0
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 1
-              - address: "127.0.0.1:9193"
+              - address_for_peers: "127.0.0.1:9193"
+                address_for_clients: "127.0.0.1:9193"
                 rack: "rack0"
                 broker_id: 2
             local_shotover_broker_id: 0

shotover-proxy/tests/test-configs/kafka/cluster-sasl-plain/topology2.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9191"
+              - address_for_peers: "127.0.0.1:9191"
+                address_for_clients: "127.0.0.1:9191"
                 rack: "rack0"
                 broker_id: 0
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 1
-              - address: "127.0.0.1:9193"
+              - address_for_peers: "127.0.0.1:9193"
+                address_for_clients: "127.0.0.1:9193"
                 rack: "rack0"
                 broker_id: 2
             local_shotover_broker_id: 1

shotover-proxy/tests/test-configs/kafka/cluster-sasl-plain/topology3.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9191"
+              - address_for_peers: "127.0.0.1:9191"
+                address_for_clients: "127.0.0.1:9191"
                 rack: "rack0"
                 broker_id: 0
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 1
-              - address: "127.0.0.1:9193"
+              - address_for_peers: "127.0.0.1:9193"
+                address_for_clients: "127.0.0.1:9193"
                 rack: "rack0"
                 broker_id: 2
             local_shotover_broker_id: 2

shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram-over-mtls/topology-single.yaml

@@ -6,7 +6,8 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 0
             local_shotover_broker_id: 0

shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram-over-mtls/topology1.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9191"
+              - address_for_peers: "127.0.0.1:9191"
+                address_for_clients: "127.0.0.1:9191"
                 rack: "rack0"
                 broker_id: 0
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 1
-              - address: "127.0.0.1:9193"
+              - address_for_peers: "127.0.0.1:9193"
+                address_for_clients: "127.0.0.1:9193"
                 rack: "rack0"
                 broker_id: 2
             local_shotover_broker_id: 0

shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram-over-mtls/topology2.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9191"
+              - address_for_peers: "127.0.0.1:9191"
+                address_for_clients: "127.0.0.1:9191"
                 rack: "rack0"
                 broker_id: 0
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 1
-              - address: "127.0.0.1:9193"
+              - address_for_peers: "127.0.0.1:9193"
+                address_for_clients: "127.0.0.1:9193"
                 rack: "rack0"
                 broker_id: 2
             local_shotover_broker_id: 1

shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram-over-mtls/topology3.yaml

@@ -6,13 +6,16 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9191"
+              - address_for_peers: "127.0.0.1:9191"
+                address_for_clients: "127.0.0.1:9191"
                 rack: "rack0"
                 broker_id: 0
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 1
-              - address: "127.0.0.1:9193"
+              - address_for_peers: "127.0.0.1:9193"
+                address_for_clients: "127.0.0.1:9193"
                 rack: "rack0"
                 broker_id: 2
             local_shotover_broker_id: 2

shotover-proxy/tests/test-configs/kafka/cluster-sasl-scram/topology-single.yaml

@@ -6,7 +6,8 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 0
             local_shotover_broker_id: 0

shotover-proxy/tests/test-configs/kafka/cluster-tls/topology.yaml

@@ -9,7 +9,8 @@ sources:
       chain:
         - KafkaSinkCluster:
             shotover_nodes:
-              - address: "127.0.0.1:9192"
+              - address_for_peers: "127.0.0.1:9192"
+                address_for_clients: "127.0.0.1:9192"
                 rack: "rack0"
                 broker_id: 0
             local_shotover_broker_id: 0