Merge branch 'main' into 1763-move-destination-to-pendingrequest

.github/workflows/build_and_test.yaml

@@ -76,7 +76,7 @@ jobs:
       # This workaround is to force j4rs to be rebuilt from scratch via cargo clean.
       # This has a cost on CI runtime and in the future we should find another solution as discussed above.
     - name: Workaround j4rs cache issue
-      run: cargo clean -p j4rs
+      run: cargo clean -p j4rs ${{ matrix.cargo_flags }}
 
     - name: Build tests
       run: |

shotover-proxy/tests/kafka_int_tests/test_cases.rs

@@ -4,8 +4,8 @@ use test_helpers::{
     connection::kafka::{
         Acl, AclOperation, AclPermissionType, AlterConfig, ConfigEntry, ConsumerConfig,
         ExpectedResponse, IsolationLevel, KafkaAdmin, KafkaConnectionBuilder, KafkaConsumer,
-        KafkaDriver, KafkaProducer, ListOffsetsResultInfo, NewPartition, NewTopic,
-        OffsetAndMetadata, OffsetSpec, Record, RecordsToDelete, ResourcePatternType,
+        KafkaDriver, KafkaProducer, ListOffsetsResultInfo, NewPartition, NewPartitionReassignment,
+        NewTopic, OffsetAndMetadata, OffsetSpec, Record, RecordsToDelete, ResourcePatternType,
         ResourceSpecifier, ResourceType, TopicPartition,
     },
     docker_compose::DockerCompose,
@@ -791,6 +791,13 @@ pub async fn produce_consume_partitions1_shotover_nodes_go_down(
                 offset: Some(0),
             })
             .await;
+        consumer.assert_commit_offsets(HashMap::from([(
+            TopicPartition {
+                topic_name: topic_name.to_owned(),
+                partition: 0,
+            },
+            1,
+        )]));
 
         // kill shotover node(s)
         for shotover_node in shotover_nodes_to_kill {
@@ -1646,6 +1653,54 @@ async fn list_transactions(connection_builder: &KafkaConnectionBuilder) {
     assert_eq!(actual_results, expected_results);
 }
 
+async fn create_and_list_partition_reassignments(connection_builder: &KafkaConnectionBuilder) {
+    let admin = connection_builder.connect_admin().await;
+    admin
+        .alter_partition_reassignments(HashMap::from([(
+            TopicPartition {
+                topic_name: "partitions1".to_owned(),
+                partition: 0,
+            },
+            NewPartitionReassignment {
+                replica_broker_ids: vec![0],
+            },
+        )]))
+        .await;
+
+    let actual_results = admin.list_partition_reassignments().await;
+
+    if actual_results.is_empty() {
+        // If too much time passes between requesting the reassignment and listing the reassignment it,
+        // the reassignment might have already completed so there is nothing to list.
+        // In that case return early to skip the assertions.
+        //
+        // The assertions should still run sometimes, so its worth keeping around.
+        // And at the very least we know that the messages are sent/received succesfully.
+        return;
+    }
+
+    assert_eq!(actual_results.len(), 1);
+    let reassignment = actual_results
+        .get(&TopicPartition {
+            topic_name: "partitions1".to_owned(),
+            partition: 0,
+        })
+        .unwrap();
+    let expected_adding_replica_broker_ids: &[i32] = if reassignment.replica_broker_ids == [0] {
+        // The original broker is randomly assigned.
+        // If it happens to be broker 0, matching the new broker we requested,
+        // then adding_replica_broker_ids will be empty
+        &[]
+    } else {
+        // otherwise it contains the new broker we requested
+        &[0]
+    };
+    assert_eq!(
+        reassignment.adding_replica_broker_ids,
+        expected_adding_replica_broker_ids
+    );
+}
+
 async fn cluster_test_suite_base(connection_builder: &KafkaConnectionBuilder) {
     let admin = connection_builder.connect_admin().await;
     admin
@@ -1672,6 +1727,7 @@ pub async fn tests_requiring_all_shotover_nodes(connection_builder: &KafkaConnec
     if let KafkaConnectionBuilder::Java(_) = connection_builder {
         list_groups(connection_builder).await;
         list_transactions(connection_builder).await;
+        create_and_list_partition_reassignments(connection_builder).await;
     }
 }
 

shotover-proxy/tests/test-configs/redis/cluster-auth/docker-compose.yaml

@@ -1,54 +1,54 @@
 services:
-  redis-node-0:
-    image: &image bitnami/redis-cluster:6.2.12-debian-11-r26
+  valkey-node-0:
+    image: &image bitnami/valkey-cluster:7.2.5-debian-12-r4
     ports:
       - "2230:6379"
     environment: &environment
-      - 'REDIS_PASSWORD=shotover'
-      - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
+      - 'VALKEY_PASSWORD=shotover'
+      - 'VALKEY_NODES=valkey-node-0 valkey-node-1 valkey-node-2 valkey-node-3 valkey-node-4 valkey-node-5'
 
-  redis-node-1:
+  valkey-node-1:
     image: *image
     ports:
       - "2231:6379"
     environment: *environment
 
-  redis-node-2:
+  valkey-node-2:
     image: *image
     ports:
       - "2232:6379"
     environment: *environment
 
-  redis-node-3:
+  valkey-node-3:
     image: *image
     ports:
       - "2233:6379"
     environment: *environment
 
-  redis-node-4:
+  valkey-node-4:
     image: *image
     ports:
       - "2234:6379"
     environment: *environment
 
-  redis-node-5:
+  valkey-node-5:
     image: *image
     ports:
       - "2235:6379"
     environment: *environment
 
-  redis-cluster-init:
+  valkey-cluster-init:
     image: *image
     depends_on:
-      - redis-node-0
-      - redis-node-1
-      - redis-node-2
-      - redis-node-3
-      - redis-node-4
-      - redis-node-5
+      - valkey-node-0
+      - valkey-node-1
+      - valkey-node-2
+      - valkey-node-3
+      - valkey-node-4
+      - valkey-node-5
     environment:
-      - 'REDIS_PASSWORD=shotover'
+      - 'VALKEY_PASSWORD=shotover'
       - 'REDISCLI_AUTH=shotover'
-      - 'REDIS_CLUSTER_REPLICAS=1'
-      - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
-      - 'REDIS_CLUSTER_CREATOR=yes'
+      - 'VALKEY_CLUSTER_REPLICAS=1'
+      - 'VALKEY_NODES=valkey-node-0 valkey-node-1 valkey-node-2 valkey-node-3 valkey-node-4 valkey-node-5'
+      - 'VALKEY_CLUSTER_CREATOR=yes'

shotover-proxy/tests/test-configs/redis/cluster-dr/docker-compose.yaml

@@ -1,108 +1,108 @@
 services:
-  redis-node-0:
-    image: &image bitnami/redis-cluster:6.2.12-debian-11-r26
+  valkey-node-0:
+    image: &image bitnami/valkey-cluster:7.2.5-debian-12-r4
     ports:
       - "2220:6379"
     environment: &node-environment
-      - 'REDIS_PASSWORD=shotover'
-      - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
+      - 'VALKEY_PASSWORD=shotover'
+      - 'VALKEY_NODES=valkey-node-0 valkey-node-1 valkey-node-2 valkey-node-3 valkey-node-4 valkey-node-5'
 
-  redis-node-1:
+  valkey-node-1:
     image: *image
     ports:
       - "2221:6379"
     environment: *node-environment
 
-  redis-node-2:
+  valkey-node-2:
     image: *image
     ports:
       - "2222:6379"
     environment: *node-environment
 
-  redis-node-3:
+  valkey-node-3:
     image: *image
     ports:
       - "2223:6379"
     environment: *node-environment
 
-  redis-node-4:
+  valkey-node-4:
     image: *image
     ports:
       - "2224:6379"
     environment: *node-environment
 
-  redis-node-5:
+  valkey-node-5:
     image: *image
     ports:
       - "2225:6379"
     environment: *node-environment
 
-  redis-cluster-init:
+  valkey-cluster-init:
     image: *image
     depends_on:
-      - redis-node-0
-      - redis-node-1
-      - redis-node-2
-      - redis-node-3
-      - redis-node-4
-      - redis-node-5
+      - valkey-node-0
+      - valkey-node-1
+      - valkey-node-2
+      - valkey-node-3
+      - valkey-node-4
+      - valkey-node-5
     environment:
-      - 'REDIS_PASSWORD=shotover'
+      - 'VALKEY_PASSWORD=shotover'
       - 'REDISCLI_AUTH=shotover'
-      - 'REDIS_CLUSTER_REPLICAS=1'
-      - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
-      - 'REDIS_CLUSTER_CREATOR=yes'
+      - 'VALKEY_CLUSTER_REPLICAS=1'
+      - 'VALKEY_NODES=valkey-node-0 valkey-node-1 valkey-node-2 valkey-node-3 valkey-node-4 valkey-node-5'
+      - 'VALKEY_CLUSTER_CREATOR=yes'
 
-  redis-node-0-dr:
+  valkey-node-0-dr:
     image: *image
     ports:
       - "2120:6379"
     environment: &dr-environment
-      - 'REDIS_PASSWORD=shotover'
-      - 'REDIS_NODES=redis-node-0-dr redis-node-1-dr redis-node-2-dr redis-node-3-dr redis-node-4-dr redis-node-5-dr'
+      - 'VALKEY_PASSWORD=shotover'
+      - 'VALKEY_NODES=valkey-node-0-dr valkey-node-1-dr valkey-node-2-dr valkey-node-3-dr valkey-node-4-dr valkey-node-5-dr'
 
-  redis-node-1-dr:
+  valkey-node-1-dr:
     image: *image
     ports:
       - "2121:6379"
     environment: *dr-environment
 
-  redis-node-2-dr:
+  valkey-node-2-dr:
     image: *image
     ports:
       - "2122:6379"
     environment: *dr-environment
 
-  redis-node-3-dr:
+  valkey-node-3-dr:
     image: *image
     ports:
       - "2123:6379"
     environment: *dr-environment
 
-  redis-node-4-dr:
+  valkey-node-4-dr:
     image: *image
     ports:
       - "2124:6379"
     environment: *dr-environment
 
-  redis-node-5-dr:
+  valkey-node-5-dr:
     image: *image
     ports:
       - "2125:6379"
     environment: *dr-environment
 
-  redis-cluster-init-dr:
+  valkey-cluster-init-dr:
     image: *image
     depends_on:
-      - redis-node-0-dr
-      - redis-node-1-dr
-      - redis-node-2-dr
-      - redis-node-3-dr
-      - redis-node-4-dr
-      - redis-node-5-dr
+      - valkey-node-0-dr
+      - valkey-node-1-dr
+      - valkey-node-2-dr
+      - valkey-node-3-dr
+      - valkey-node-4-dr
+      - valkey-node-5-dr
     environment:
-      - 'REDIS_PASSWORD=shotover'
+      - 'VALKEY_PASSWORD=shotover'
       - 'REDISCLI_AUTH=shotover'
-      - 'REDIS_CLUSTER_REPLICAS=1'
-      - 'REDIS_NODES=redis-node-0-dr redis-node-1-dr redis-node-2-dr redis-node-3-dr redis-node-4-dr redis-node-5-dr'
-      - 'REDIS_CLUSTER_CREATOR=yes'
+      - 'VALKEY_CLUSTER_REPLICAS=1'
+      - 'VALKEY_NODES=valkey-node-0-dr valkey-node-1-dr valkey-node-2-dr valkey-node-3-dr valkey-node-4-dr valkey-node-5-dr'
+      - 'VALKEY_CLUSTER_CREATOR=yes'