Compress datasets.json

crates/network-scheduler/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "network-scheduler"
-version = "1.0.23"
+version = "1.0.24"
 edition = "2021"
 
 [dependencies]
@@ -14,6 +14,7 @@ clap = { version = "4", features = ["derive", "env"] }
 dashmap = { version = "6", features = ["serde"] }
 derive-enum-from-into = "0.1"
 env_logger = "0.11"
+flate2 = "1.0"
 futures = "0.3"
 hex = "0.4"
 hmac = "0.12.1"

crates/network-scheduler/src/storage.rs

@@ -232,7 +232,7 @@ impl S3Storage {
             .await;
         let client = s3::Client::new(&s3_config);
         let scheduler_state_key = format!("scheduler_{scheduler_id}.json");
-        let chunks_list_key = format!("datasets_{}.json", config.network);
+        let chunks_list_key = format!("datasets_{}.json.gz", config.network);
         Self {
             client,
             config,
@@ -324,13 +324,16 @@ impl S3Storage {
         let json = serde_json::json!({
             "chunks": chunks
         });
-        let future = match serde_json::to_vec(&json) {
-            Ok(bytes) => Some(
+        let future = match serde_json::to_vec(&json)
+            .map_err(anyhow::Error::from)
+            .and_then(gzip)
+        {
+            Ok(compressed) => Some(
                 self.client
                     .put_object()
                     .bucket(&self.config.scheduler_state_bucket)
                     .key(&self.chunks_list_key)
-                    .body(bytes.into())
+                    .body(compressed.into())
                     .send(),
             ),
             Err(e) => {
@@ -347,3 +350,11 @@ impl S3Storage {
         }
     }
 }
+
+fn gzip(data: Vec<u8>) -> anyhow::Result<Vec<u8>> {
+    use flate2::write::GzEncoder;
+    use std::io::Write;
+    let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default());
+    encoder.write_all(&data)?;
+    Ok(encoder.finish()?)
+}