Add feature flags for each protocol in shotover

.cargo/config.toml

@@ -8,6 +8,17 @@ rustflags = [
 linker = "aarch64-linux-gnu-gcc"
 
 [alias]
-windsock = "test --release --bench windsock --features alpha-transforms,rdkafka-driver-tests --"
-windsock-debug = "test --bench windsock --features alpha-transforms,rdkafka-driver-tests --"
-windsock-cloud-docker = "run --package windsock-cloud-docker --"
+# Can run every benchmark
+windsock = "test --release --bench windsock --features kafka,alpha-transforms,rdkafka-driver-tests,cassandra,redis --"
+windsock-debug = "test --bench windsock --features kafka,alpha-transforms,rdkafka-driver-tests,cassandra,redis --"
+
+# Can only run benchmarks specific to the protocol but compiles a lot faster
+windsock-redis = "test --release --bench windsock --no-default-features --features redis,alpha-transforms --"
+windsock-kafka = "test --release --bench windsock --no-default-features --features kafka,alpha-transforms,rdkafka-driver-tests --"
+windsock-cassandra = "test --release --bench windsock --no-default-features --features cassandra,alpha-transforms --"
+
+# Compile benches in docker to ensure compiled libc version is compatible with the EC2 instances libc
+windsock-cloud-docker = "run --package windsock-cloud-docker -- redis,cassandra,kafka"
+windsock-cloud-docker-redis = "run --package windsock-cloud-docker -- redis"
+windsock-cloud-docker-kafka = "run --package windsock-cloud-docker -- kafka"
+windsock-cloud-docker-cassandra = "run --package windsock-cloud-docker -- cassandra"

.github/workflows/build.yaml

@@ -40,11 +40,13 @@ jobs:
       - name: Install ubuntu packages
         run: shotover-proxy/build/install_ubuntu_packages.sh
       - name: Install cargo-hack
-        run: cargo install cargo-hack --version 0.5.8
+        uses: taiki-e/install-action@v2
+        with:
+          tool: [email protected]
       - name: Ensure that dev tools compiles and has no warnings with no features enabled
         run: cargo clippy --locked ${{ matrix.cargo_flags }} --all-targets -- -D warnings
       - name: Ensure that shotover-proxy compiles and has no warnings under every possible combination of features
         # some things to explicitly point out:
         # * clippy also reports rustc warnings and errors
         # * clippy --all-targets is not run so we only build the shotover_proxy executable without the tests/benches
-        run: cargo hack --feature-powerset clippy --locked ${{ matrix.cargo_flags }} --package shotover-proxy -- -D warnings
+        run: cargo hack --feature-powerset --at-least-one-of redis,cassandra,kafka,opensearch clippy --locked ${{ matrix.cargo_flags }} --package shotover-proxy -- -D warnings

.github/workflows/lint.yaml

@@ -40,14 +40,14 @@ jobs:
     - name: Install cargo-hack
       uses: taiki-e/install-action@v2
       with:
-        tool: [email protected].4
+        tool: [email protected].16
     - name: Ensure `cargo fmt --all` was run
       run: cargo fmt --all -- --check
     - name: Ensure that all crates compile and have no warnings under every possible combination of features
       # some things to explicitly point out:
       # * clippy also reports rustc warnings and errors
       # * clippy --all-targets causes clippy to run against tests and examples which it doesnt do by default.
-      run: cargo hack --feature-powerset clippy --all-targets --locked -- -D warnings
+      run: cargo hack --feature-powerset --at-least-one-of redis,cassandra,kafka,opensearch clippy --all-targets --locked -- -D warnings
     - name: Report disk usage
       run: |
         df -h

Cargo.toml

@@ -30,7 +30,7 @@ codegen-units = 1
 scylla = { version = "0.11.0", features = ["ssl"] }
 bytes = "1.0.0"
 tokio = { version = "1.25.0", features = ["full", "macros"] }
-tokio-util = { version = "0.7.7" }
+tokio-util = { version = "0.7.7", features = ["codec"] }
 tokio-openssl = "0.6.2"
 itertools = "0.12.0"
 openssl = { version = "0.10.36", features = ["vendored"] }

custom-transforms-example/Cargo.toml

@@ -8,7 +8,7 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-shotover = { path = "../shotover" }
+shotover = { path = "../shotover", default-features = false}
 anyhow.workspace = true
 serde.workspace = true
 async-trait.workspace = true
@@ -20,3 +20,11 @@ typetag.workspace = true
 test-helpers = {path = "../test-helpers"}
 tokio.workspace = true
 redis.workspace = true
+
+[features]
+redis = ["shotover/redis"]
+default = ["redis"]
+
+[[test]]
+name = "test"
+required-features = ["redis"]

custom-transforms-example/src/main.rs

@@ -1,6 +1,8 @@
 use shotover::runner::Shotover;
 
+#[cfg(feature = "redis")]
 mod redis_get_rewrite;
+#[cfg(feature = "redis")]
 shotover::import_transform!(redis_get_rewrite::RedisGetRewriteConfig);
 
 fn main() {

shotover-proxy/Cargo.toml

@@ -8,7 +8,7 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-shotover = { path = "../shotover" }
+shotover = { path = "../shotover", default-features = false}
 
 [dev-dependencies]
 prometheus-parse = "0.2.4"
@@ -60,8 +60,13 @@ shell-quote.workspace = true
 [features]
 # Include WIP alpha transforms in the public API
 alpha-transforms = ["shotover/alpha-transforms"]
+cassandra = ["shotover/cassandra"]
+kafka = ["shotover/kafka"]
+redis = ["shotover/redis"]
+opensearch = ["shotover/opensearch"]
 cassandra-cpp-driver-tests = ["test-helpers/cassandra-cpp-driver-tests"]
 rdkafka-driver-tests = ["test-helpers/rdkafka-driver-tests"]
+default = ["cassandra", "kafka", "redis", "opensearch"]
 
 [[bench]]
 name = "windsock"

shotover-proxy/benches/windsock/cloud/aws.rs

@@ -255,7 +255,7 @@ sudo docker system prune -af"#,
         }
     }
 
-    #[cfg(feature = "rdkafka-driver-tests")]
+    #[cfg(all(feature = "rdkafka-driver-tests", feature = "kafka"))]
     pub async fn run_shotover(self: Arc<Self>, topology: &str) -> RunningShotover {
         self.instance
             .ssh()

shotover-proxy/benches/windsock/main.rs

@@ -1,19 +1,33 @@
+// Allow dead code if any of the protocol features are disabled
+#![cfg_attr(
+    any(
+        not(feature = "cassandra"),
+        not(feature = "redis"),
+        not(all(feature = "rdkafka-driver-tests", feature = "kafka"))
+    ),
+    allow(dead_code, unused_imports)
+)]
+
+#[cfg(feature = "cassandra")]
 mod cassandra;
 mod cloud;
 mod common;
-#[cfg(feature = "rdkafka-driver-tests")]
+#[cfg(all(feature = "rdkafka-driver-tests", feature = "kafka"))]
 mod kafka;
 mod profilers;
+#[cfg(feature = "redis")]
 mod redis;
 mod shotover;
 
-use crate::cassandra::*;
-use crate::common::*;
-#[cfg(feature = "rdkafka-driver-tests")]
-use crate::kafka::*;
-use crate::redis::*;
+#[cfg(feature = "cassandra")]
+use cassandra::*;
 use cloud::CloudResources;
 use cloud::CloudResourcesRequired;
+use common::*;
+#[cfg(all(feature = "rdkafka-driver-tests", feature = "kafka"))]
+use kafka::*;
+#[cfg(feature = "redis")]
+use redis::*;
 use std::path::Path;
 use tracing_subscriber::EnvFilter;
 use windsock::{Bench, Windsock};
@@ -22,7 +36,7 @@ type ShotoverBench = Box<
     dyn Bench<CloudResourcesRequired = CloudResourcesRequired, CloudResources = CloudResources>,
 >;
 
-fn main() {
+pub fn main() {
     let (non_blocking, _guard) = tracing_appender::non_blocking(std::io::stdout());
     tracing_subscriber::fmt()
         .with_env_filter(EnvFilter::from_default_env())
@@ -41,6 +55,7 @@ fn main() {
         .unwrap();
     }
 
+    #[cfg(feature = "cassandra")]
     let cassandra_benches = itertools::iproduct!(
         [CassandraDb::Cassandra],
         [CassandraTopology::Single, CassandraTopology::Cluster3],
@@ -88,7 +103,36 @@ fn main() {
             )) as ShotoverBench)
         },
     );
-    #[cfg(feature = "rdkafka-driver-tests")]
+    #[cfg(not(feature = "cassandra"))]
+    let cassandra_benches = std::iter::empty();
+    #[cfg(feature = "cassandra")]
+    let cassandra_mock_benches = vec![
+        Box::new(CassandraBench::new(
+            CassandraDb::Mocked,
+            CassandraTopology::Single,
+            Shotover::None,
+            Compression::None,
+            Operation::ReadI64,
+            CassandraProtocol::V4,
+            CassandraDriver::Scylla,
+            10,
+        )) as ShotoverBench,
+        Box::new(CassandraBench::new(
+            CassandraDb::Mocked,
+            CassandraTopology::Single,
+            Shotover::Standard,
+            Compression::None,
+            Operation::ReadI64,
+            CassandraProtocol::V4,
+            CassandraDriver::Scylla,
+            10,
+        )),
+    ]
+    .into_iter();
+    #[cfg(not(feature = "cassandra"))]
+    let cassandra_mock_benches = std::iter::empty();
+
+    #[cfg(all(feature = "rdkafka-driver-tests", feature = "kafka"))]
     let kafka_benches = itertools::iproduct!(
         [
             Shotover::None,
@@ -105,9 +149,10 @@ fn main() {
     .map(|(shotover, topology, size)| {
         Box::new(KafkaBench::new(shotover, topology, size)) as ShotoverBench
     });
-    #[cfg(not(feature = "rdkafka-driver-tests"))]
+    #[cfg(not(all(feature = "rdkafka-driver-tests", feature = "kafka")))]
     let kafka_benches = std::iter::empty();
 
+    #[cfg(feature = "redis")]
     let redis_benches = itertools::iproduct!(
         [RedisTopology::Cluster3, RedisTopology::Single],
         [
@@ -121,35 +166,15 @@ fn main() {
     .map(|(topology, shotover, operation, encryption)| {
         Box::new(RedisBench::new(topology, shotover, operation, encryption)) as ShotoverBench
     });
+    #[cfg(not(feature = "redis"))]
+    let redis_benches = std::iter::empty();
 
     Windsock::new(
-        vec![
-            Box::new(CassandraBench::new(
-                CassandraDb::Mocked,
-                CassandraTopology::Single,
-                Shotover::None,
-                Compression::None,
-                Operation::ReadI64,
-                CassandraProtocol::V4,
-                CassandraDriver::Scylla,
-                10,
-            )) as ShotoverBench,
-            Box::new(CassandraBench::new(
-                CassandraDb::Mocked,
-                CassandraTopology::Single,
-                Shotover::Standard,
-                Compression::None,
-                Operation::ReadI64,
-                CassandraProtocol::V4,
-                CassandraDriver::Scylla,
-                10,
-            )),
-        ]
-        .into_iter()
-        .chain(cassandra_benches)
-        .chain(kafka_benches)
-        .chain(redis_benches)
-        .collect(),
+        cassandra_mock_benches
+            .chain(cassandra_benches)
+            .chain(kafka_benches)
+            .chain(redis_benches)
+            .collect(),
         cloud::AwsCloud::new_boxed(),
         &["release"],
     )

shotover-proxy/tests/lib.rs

@@ -1,15 +1,20 @@
-#[allow(clippy::single_component_path_imports)]
+#[allow(clippy::single_component_path_imports, unused_imports)]
 use rstest_reuse;
 
 use test_helpers::shotover_process::ShotoverProcessBuilder;
 use tokio_bin_process::bin_path;
 
+#[cfg(feature = "cassandra")]
 mod cassandra_int_tests;
+#[cfg(feature = "kafka")]
 mod kafka_int_tests;
-#[cfg(feature = "alpha-transforms")]
+#[cfg(all(feature = "alpha-transforms", feature = "opensearch"))]
 mod opensearch_int_tests;
+#[cfg(feature = "redis")]
 mod redis_int_tests;
+#[cfg(feature = "redis")]
 mod runner;
+#[cfg(feature = "redis")]
 mod transforms;
 
 pub fn shotover_process(topology_path: &str) -> ShotoverProcessBuilder {