Add feature flags for each protocol in shotover

.cargo/config.toml

@@ -8,6 +8,17 @@ rustflags = [
 linker = "aarch64-linux-gnu-gcc"
 
 [alias]
-windsock = "test --release --bench windsock --features alpha-transforms,rdkafka-driver-tests --"
-windsock-debug = "test --bench windsock --features alpha-transforms,rdkafka-driver-tests --"
-windsock-cloud-docker = "run --package windsock-cloud-docker --"
+# Can run every benchmark
+windsock = "test --release --bench windsock --features kafka,alpha-transforms,rdkafka-driver-tests,cassandra,redis --"
+windsock-debug = "test --bench windsock --features kafka,alpha-transforms,rdkafka-driver-tests,cassandra,redis --"
+
+# Can only run benchmarks specific to the protocol but compiles a lot faster
+windsock-redis = "test --release --bench windsock --no-default-features --features redis,alpha-transforms --"
+windsock-kafka = "test --release --bench windsock --no-default-features --features kafka,alpha-transforms,rdkafka-driver-tests --"
+windsock-cassandra = "test --release --bench windsock --no-default-features --features cassandra,alpha-transforms --"
+
+# Compile benches in docker to ensure compiled libc version is compatible with the EC2 instances libc
+windsock-cloud-docker = "run --package windsock-cloud-docker -- redis,cassandra,kafka"
+windsock-cloud-docker-redis = "run --package windsock-cloud-docker -- redis"
+windsock-cloud-docker-kafka = "run --package windsock-cloud-docker -- kafka"
+windsock-cloud-docker-cassandra = "run --package windsock-cloud-docker -- cassandra"

.github/workflows/build.yaml

@@ -40,11 +40,13 @@ jobs:
       - name: Install ubuntu packages
         run: shotover-proxy/build/install_ubuntu_packages.sh
       - name: Install cargo-hack
-        run: cargo install cargo-hack --version 0.5.8
+        uses: taiki-e/install-action@v2
+        with:
+          tool: [email protected]
       - name: Ensure that dev tools compiles and has no warnings with no features enabled
         run: cargo clippy --locked ${{ matrix.cargo_flags }} --all-targets -- -D warnings
       - name: Ensure that shotover-proxy compiles and has no warnings under every possible combination of features
         # some things to explicitly point out:
         # * clippy also reports rustc warnings and errors
         # * clippy --all-targets is not run so we only build the shotover_proxy executable without the tests/benches
-        run: cargo hack --feature-powerset clippy --locked ${{ matrix.cargo_flags }} --package shotover-proxy -- -D warnings
+        run: cargo hack --feature-powerset --at-least-one-of redis,cassandra,kafka,opensearch clippy --locked ${{ matrix.cargo_flags }} --package shotover-proxy -- -D warnings

.github/workflows/lint.yaml

@@ -40,14 +40,14 @@ jobs:
     - name: Install cargo-hack
       uses: taiki-e/install-action@v2
       with:
-        tool: [email protected].4
+        tool: [email protected].16
     - name: Ensure `cargo fmt --all` was run
       run: cargo fmt --all -- --check
     - name: Ensure that all crates compile and have no warnings under every possible combination of features
       # some things to explicitly point out:
       # * clippy also reports rustc warnings and errors
       # * clippy --all-targets causes clippy to run against tests and examples which it doesnt do by default.
-      run: cargo hack --feature-powerset clippy --all-targets --locked -- -D warnings
+      run: cargo hack --feature-powerset --at-least-one-of redis,cassandra,kafka,opensearch clippy --all-targets --locked -- -D warnings
     - name: Report disk usage
       run: |
         df -h

Cargo.toml

@@ -30,7 +30,7 @@ codegen-units = 1
 scylla = { version = "0.11.0", features = ["ssl"] }
 bytes = "1.0.0"
 tokio = { version = "1.25.0", features = ["full", "macros"] }
-tokio-util = { version = "0.7.7" }
+tokio-util = { version = "0.7.7", features = ["codec"] }
 tokio-openssl = "0.6.2"
 itertools = "0.12.0"
 openssl = { version = "0.10.36", features = ["vendored"] }

custom-transforms-example/Cargo.toml

@@ -8,7 +8,7 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-shotover = { path = "../shotover" }
+shotover = { path = "../shotover", default-features = false}
 anyhow.workspace = true
 serde.workspace = true
 async-trait.workspace = true
@@ -20,3 +20,7 @@ typetag.workspace = true
 test-helpers = {path = "../test-helpers"}
 tokio.workspace = true
 redis.workspace = true
+
+[features]
+redis = ["shotover/redis"]
+default = ["redis"]

custom-transforms-example/src/main.rs

@@ -1,6 +1,8 @@
 use shotover::runner::Shotover;
 
+#[cfg(feature = "redis")]
 mod redis_get_rewrite;
+#[cfg(feature = "redis")]
 shotover::import_transform!(redis_get_rewrite::RedisGetRewriteConfig);
 
 fn main() {

custom-transforms-example/tests/redis.rs

@@ -0,0 +1,66 @@
+use redis::aio::Connection;
+use redis::Cmd;
+use std::time::Duration;
+use test_helpers::connection::redis_connection;
+use test_helpers::docker_compose::docker_compose;
+use test_helpers::shotover_process::{bin_path, BinProcess, EventMatcher, Level};
+
+#[tokio::test(flavor = "multi_thread")]
+async fn test_custom_transform() {
+    // Setup shotover and the redis server it connects to
+    let _compose = docker_compose("config/docker-compose.yaml");
+    let shotover = shotover_proxy("config/topology.yaml").await;
+    let mut connection = redis_connection::new_async("127.0.0.1", 6379).await;
+
+    // Verify functionality of transform
+    assert_ok(
+        redis::cmd("SET").arg("foo").arg("some value"),
+        &mut connection,
+    )
+    .await;
+    assert_bytes(
+        redis::cmd("GET").arg("foo"),
+        &mut connection,
+        b"Rewritten value",
+    )
+    .await;
+    assert_bytes(
+        redis::cmd("GET").arg("bar"),
+        &mut connection,
+        b"Rewritten value",
+    )
+    .await;
+
+    // Shutdown shotover asserting that it encountered no errors
+    shotover.shutdown_and_then_consume_events(&[]).await;
+}
+
+async fn shotover_proxy(topology_path: &str) -> BinProcess {
+    let mut shotover = BinProcess::start_binary(
+        bin_path!("custom-transforms-example"),
+        "shotover",
+        &["-t", topology_path, "--log-format", "json"],
+    )
+    .await;
+
+    tokio::time::timeout(
+        Duration::from_secs(30),
+        shotover.wait_for(
+            &EventMatcher::new()
+                .with_level(Level::Info)
+                .with_message("Shotover is now accepting inbound connections"),
+            &[],
+        ),
+    )
+    .await
+    .unwrap();
+    shotover
+}
+
+pub async fn assert_ok(cmd: &mut Cmd, connection: &mut Connection) {
+    assert_eq!(cmd.query_async(connection).await, Ok("OK".to_string()));
+}
+
+pub async fn assert_bytes(cmd: &mut Cmd, connection: &mut Connection, value: &[u8]) {
+    assert_eq!(cmd.query_async(connection).await, Ok(value.to_vec()));
+}

custom-transforms-example/tests/test.rs

@@ -1,66 +1,2 @@
-use redis::aio::Connection;
-use redis::Cmd;
-use std::time::Duration;
-use test_helpers::connection::redis_connection;
-use test_helpers::docker_compose::docker_compose;
-use test_helpers::shotover_process::{bin_path, BinProcess, EventMatcher, Level};
-
-#[tokio::test(flavor = "multi_thread")]
-async fn test_custom_transform() {
-    // Setup shotover and the redis server it connects to
-    let _compose = docker_compose("config/docker-compose.yaml");
-    let shotover = shotover_proxy("config/topology.yaml").await;
-    let mut connection = redis_connection::new_async("127.0.0.1", 6379).await;
-
-    // Verify functionality of transform
-    assert_ok(
-        redis::cmd("SET").arg("foo").arg("some value"),
-        &mut connection,
-    )
-    .await;
-    assert_bytes(
-        redis::cmd("GET").arg("foo"),
-        &mut connection,
-        b"Rewritten value",
-    )
-    .await;
-    assert_bytes(
-        redis::cmd("GET").arg("bar"),
-        &mut connection,
-        b"Rewritten value",
-    )
-    .await;
-
-    // Shutdown shotover asserting that it encountered no errors
-    shotover.shutdown_and_then_consume_events(&[]).await;
-}
-
-async fn shotover_proxy(topology_path: &str) -> BinProcess {
-    let mut shotover = BinProcess::start_binary(
-        bin_path!("custom-transforms-example"),
-        "shotover",
-        &["-t", topology_path, "--log-format", "json"],
-    )
-    .await;
-
-    tokio::time::timeout(
-        Duration::from_secs(30),
-        shotover.wait_for(
-            &EventMatcher::new()
-                .with_level(Level::Info)
-                .with_message("Shotover is now accepting inbound connections"),
-            &[],
-        ),
-    )
-    .await
-    .unwrap();
-    shotover
-}
-
-pub async fn assert_ok(cmd: &mut Cmd, connection: &mut Connection) {
-    assert_eq!(cmd.query_async(connection).await, Ok("OK".to_string()));
-}
-
-pub async fn assert_bytes(cmd: &mut Cmd, connection: &mut Connection, value: &[u8]) {
-    assert_eq!(cmd.query_async(connection).await, Ok(value.to_vec()));
-}
+#[cfg(feature = "redis")]
+mod redis;

shotover-proxy/Cargo.toml

@@ -8,7 +8,7 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-shotover = { path = "../shotover" }
+shotover = { path = "../shotover", default-features = false}
 
 [dev-dependencies]
 prometheus-parse = "0.2.4"
@@ -60,8 +60,13 @@ shell-quote.workspace = true
 [features]
 # Include WIP alpha transforms in the public API
 alpha-transforms = ["shotover/alpha-transforms"]
+cassandra = ["shotover/cassandra"]
+kafka = ["shotover/kafka"]
+redis = ["shotover/redis"]
+opensearch = ["shotover/opensearch"]
 cassandra-cpp-driver-tests = ["test-helpers/cassandra-cpp-driver-tests"]
 rdkafka-driver-tests = ["test-helpers/rdkafka-driver-tests"]
+default = ["cassandra", "kafka", "redis", "opensearch"]
 
 [[bench]]
 name = "windsock"

shotover-proxy/benches/windsock/cloud/aws.rs

@@ -255,7 +255,7 @@ sudo docker system prune -af"#,
         }
     }
 
-    #[cfg(feature = "rdkafka-driver-tests")]
+    #[cfg(all(feature = "rdkafka-driver-tests", feature = "kafka"))]
     pub async fn run_shotover(self: Arc<Self>, topology: &str) -> RunningShotover {
         self.instance
             .ssh()