windsock: add --load-cloud-resources-from-disk and --store-cloud-reso…

…urces-to-disk

Cargo.toml

@@ -61,7 +61,7 @@ rand_distr = "0.4.1"
 clap = { version = "4.0.4", features = ["cargo", "derive"] }
 async-trait = "0.1.30"
 typetag = "0.2.5"
-aws-throwaway = { version = "0.6.0", default-features = false }
+aws-throwaway = { version = "0.6.0", default-features = false, branch = "allow_reusing_instances", git = "https://github.com/rukai/aws-throwaway" }
 tokio-bin-process = "0.4.0"
 ordered-float = { version = "4.0.0", features = ["serde"] }
 hyper = { version = "0.14.14", features = ["server"] }

shotover-proxy/benches/windsock/cloud/aws.rs

@@ -1,6 +1,7 @@
 use aws_throwaway::{Aws, Ec2Instance, InstanceType};
 use aws_throwaway::{CleanupResources, Ec2InstanceDefinition};
 use regex::Regex;
+use serde::{Deserialize, Serialize};
 use std::fmt::Write;
 use std::time::Duration;
 use std::{
@@ -175,6 +176,7 @@ sudo apt-get install -y sysstat"#,
 }
 
 /// Despite the name can also run shotover
+#[derive(Serialize, Deserialize)]
 pub struct Ec2InstanceWithDocker {
     pub instance: Ec2Instance,
 }
@@ -275,6 +277,7 @@ fn get_compatible_instance_type() -> InstanceType {
     }
 }
 
+#[derive(Serialize, Deserialize)]
 pub struct Ec2InstanceWithBencher {
     pub instance: Ec2Instance,
 }
@@ -292,6 +295,7 @@ impl Ec2InstanceWithBencher {
     }
 }
 
+#[derive(Serialize, Deserialize)]
 pub struct Ec2InstanceWithShotover {
     pub instance: Ec2Instance,
 }

shotover-proxy/benches/windsock/cloud/mod.rs

@@ -8,7 +8,8 @@ pub use aws::{
 
 use async_trait::async_trait;
 use aws::AwsInstances;
-use std::sync::Arc;
+use serde::{Deserialize, Serialize};
+use std::{path::Path, sync::Arc};
 use windsock::cloud::{BenchInfo, Cloud};
 
 pub struct AwsCloud {
@@ -62,6 +63,7 @@ impl Cloud for AwsCloud {
             shotover,
             docker,
             bencher,
+            required,
         }
     }
 
@@ -97,9 +99,62 @@ impl Cloud for AwsCloud {
 
         // TODO: spin up background tokio task to delete unneeded EC2 instances once we add the functionality to aws_throwaway
     }
+
+    async fn store_resources_to_disk(&mut self, path: &Path, resources: CloudResources) {
+        let resources = CloudResourcesDisk {
+            shotover: resources
+                .shotover
+                .into_iter()
+                .map(|x| {
+                    Arc::into_inner(x)
+                        .expect("A bench is still referencing an Ec2InstanceWithShotover")
+                })
+                .collect(),
+            docker: resources
+                .docker
+                .into_iter()
+                .map(|x| {
+                    Arc::into_inner(x)
+                        .expect("A bench is still referencing an Ec2InstanceWithDocker")
+                })
+                .collect(),
+            bencher: resources.bencher.map(|x| {
+                Arc::into_inner(x).expect("A bench is still referencing an Ec2InstanceWithBencher")
+            }),
+            required: resources.required,
+        };
+        std::fs::write(path, serde_json::to_string(&resources).unwrap()).unwrap();
+    }
+
+    async fn load_resources_from_disk(
+        &mut self,
+        path: &Path,
+        required_resources: Vec<CloudResourcesRequired>,
+    ) -> CloudResources {
+        let required_resources = required_resources.into_iter().fold(
+            CloudResourcesRequired::default(),
+            CloudResourcesRequired::combine,
+        );
+        let resources: CloudResourcesDisk =
+            serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();
+        if resources.required != required_resources {
+            panic!(
+                "Stored resources do not meet the requirements of this bench run. Maybe try rerunning --store-cloud-resources-to-disk\nloaded:\n{:?}\nrequired:\n{:?}",
+                resources.required,
+                required_resources
+            );
+        }
+
+        CloudResources {
+            shotover: resources.shotover.into_iter().map(Arc::new).collect(),
+            docker: resources.docker.into_iter().map(Arc::new).collect(),
+            bencher: resources.bencher.map(Arc::new),
+            required: resources.required,
+        }
+    }
 }
 
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
 pub struct CloudResourcesRequired {
     pub shotover_instance_count: usize,
     pub docker_instance_count: usize,
@@ -125,4 +180,13 @@ pub struct CloudResources {
     pub shotover: Vec<Arc<Ec2InstanceWithShotover>>,
     pub docker: Vec<Arc<Ec2InstanceWithDocker>>,
     pub bencher: Option<Arc<Ec2InstanceWithBencher>>,
+    pub required: CloudResourcesRequired,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct CloudResourcesDisk {
+    pub shotover: Vec<Ec2InstanceWithShotover>,
+    pub docker: Vec<Ec2InstanceWithDocker>,
+    pub bencher: Option<Ec2InstanceWithBencher>,
+    pub required: CloudResourcesRequired,
 }

windsock/src/cli.rs

@@ -68,6 +68,18 @@ pub struct Args {
     #[clap(long, verbatim_doc_comment)]
     pub cleanup_cloud_resources: bool,
 
+    /// Skip running of benches.
+    /// Skip automatic deletion of cloud resources on bench run completion.
+    /// Instead, just create cloud resources and write details of the resources to disk so they may be restored via `--load-cloud-resources-from-disk`
+    #[clap(long, verbatim_doc_comment)]
+    pub store_cloud_resources_to_disk: bool,
+
+    /// Skip automatic creation of cloud resources on bench run completion.
+    /// Skip automatic deletion of cloud resources on bench run completion.
+    /// Instead, details of the resources are loaded from disk as saved via a previous run using `--store-cloud-resources-to-disk`
+    #[clap(long, verbatim_doc_comment)]
+    pub load_cloud_resources_from_disk: bool,
+
     /// Display results from the last benchmark run by:
     ///     Comparing various benches against a specific base bench.
     ///

windsock/src/cloud.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 use async_trait::async_trait;
 
 /// Implement this to give windsock some control over your cloud.
@@ -21,6 +23,24 @@ pub trait Cloud {
         required_resources: Vec<Self::CloudResourcesRequired>,
     ) -> Self::CloudResources;
 
+    /// Construct a file at the provided path that will allow restoring the passed resources
+    ///
+    /// It is gauranteed this will be called after all the benches have completed.
+    async fn store_resources_to_disk(&mut self, path: &Path, resources: Self::CloudResources);
+
+    /// Restore the resources from the data in the passed file.
+    /// It is the same file path that was passed to [`Cloud::store_resources_to_disk`]
+    ///
+    /// The implementation should panic when the loaded messages cannot meet the requirements of the passed `required_sources`.
+    /// This is done rather than loading the required resources from disk as this case usually represents a user error.
+    /// Loading from disk is used for more consistent results across benches but the user cannot hope to get consistent results while changing the benches that will be run.
+    /// They are better off recreating the resources from scratch in this case.
+    async fn load_resources_from_disk(
+        &mut self,
+        path: &Path,
+        required_resources: Vec<Self::CloudResourcesRequired>,
+    ) -> Self::CloudResources;
+
     /// This is called once at start up before running any benches.
     /// The returned Vec specifies the order in which to run benches.
     fn order_benches(
@@ -32,6 +52,7 @@ pub trait Cloud {
 
     /// This is called before running each bench.
     /// Use it to destroy or create resources as needed.
+    /// However, this method will not be called when `--save-resources-to-disk` or `--load-resources-from-disk` is set.
     ///
     /// It is recommended to create all resources within create_resources for faster completion time, but it may be desirable in some circumstances to create some of them here.
     /// It is recommended to always destroy resources that will never be used again here.
@@ -66,4 +87,11 @@ impl Cloud for NoCloud {
     type CloudResources = ();
     async fn cleanup_resources(&mut self) {}
     async fn create_resources(&mut self, _requests: Vec<()>) {}
+    async fn store_resources_to_disk(&mut self, _path: &Path, _resources: ()) {}
+    async fn load_resources_from_disk(
+        &mut self,
+        _path: &Path,
+        _required_resources: Vec<Self::CloudResourcesRequired>,
+    ) {
+    }
 }

windsock/src/data.rs

@@ -12,3 +12,7 @@ pub fn windsock_path() -> PathBuf {
 
     PathBuf::from("windsock_data")
 }
+
+pub fn cloud_resources_path() -> PathBuf {
+    windsock_path().join("cloud_resources")
+}