From e8c6b0c40b0d68772215632fa1cf09286ed93393 Mon Sep 17 00:00:00 2001 From: imDema Date: Thu, 14 Mar 2024 15:33:29 +0100 Subject: [PATCH] Switch to toml for deployment config --- Cargo.lock | 61 +++++++++++++++++++++++++++++++---------- Cargo.toml | 5 ++-- README.md | 26 +++++++++--------- examples/nexmark.rs | 6 ++-- src/config.rs | 31 +++++++++++---------- src/lib.rs | 26 +++++++++--------- src/network/topology.rs | 18 ++++++------ src/runner.rs | 4 +-- 8 files changed, 107 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f822aa4..687e68a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -913,13 +913,13 @@ dependencies = [ "regex", "serde", "serde_json", - "serde_yaml", "sha2", "shell-escape", "ssh2", "tempfile", "thiserror", "tokio", + "toml", "tracing", "tracing-subscriber", "typemap_rev", @@ -1294,16 +1294,12 @@ dependencies = [ ] [[package]] -name = "serde_yaml" -version = "0.9.32" +name = "serde_spanned" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd075d994154d4a774f95b51fb96bdc2832b0ea48425c92546073816cda1f2f" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" dependencies = [ - "indexmap", - "itoa", - "ryu", "serde", - "unsafe-libyaml", ] [[package]] @@ -1492,6 +1488,40 @@ dependencies = [ "syn 2.0.52", ] +[[package]] +name = "toml" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af06656561d28735e9c1cd63dfd57132c8155426aa6af24f36a00a351f88c48e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18769cd1cec395d70860ceb4d932812a0b4d06b1a4bb336745a4d21b9496e992" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tracing" version = "0.1.40" @@ -1568,12 +1598,6 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" -[[package]] -name = "unsafe-libyaml" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" - [[package]] name = "utf8parse" version = "0.2.1" @@ -1867,6 +1891,15 @@ version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +[[package]] +name = "winnow" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" +dependencies = [ + "memchr", +] + [[package]] name = "wyhash" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index af41f636..cb67a3f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,12 +33,11 @@ nanorand = "0.7.0" # utility macros for customizing the derive Debug, Default, ... derivative = "2.2.0" -# serialization library used for reading the config file (yaml) and serializing -# the messages on the network +# serialization serde = { version = "1.0.197", features = ["derive"] } -serde_yaml = "0.9.32" serde_json = "1.0.114" bincode = "1.3.3" +toml = "0.8.11" # handy Result type anyhow = "1.0.81" diff --git a/README.md b/README.md index fcc5e933..296528dd 100644 --- a/README.md +++ b/README.md @@ -90,23 +90,23 @@ fn tokenize(s: &str) -> Vec { s.split_whitespace().map(str::to_lowercase).collect() } -// Execute on multiple hosts `cargo run -- -r config.yaml input.txt` +// Execute on multiple hosts `cargo run -- -r config.toml input.txt` ``` ### Remote deployment -```yaml -# config.yaml -hosts: - - address: host1.lan - base_port: 9500 - num_cores: 16 - - address: host2.lan - base_port: 9500 - num_cores: 8 - ssh: - username: noir-compute - key_file: /home/user/.ssh/id_rsa +```toml +# config.toml +[[host]] +address = "host1.lan" +base_port = 9500 +num_cores = 16 + +[[host]] +address = "host2.lan" +base_port = 9500 +num_cores = 24 +ssh = { username = "noir", key_file = "/home/noir/.ssh/id_ed25519" } ``` Refer to the [examples](examples/) directory for an extended set of working examples diff --git a/examples/nexmark.rs b/examples/nexmark.rs index f86248ba..596b21c5 100644 --- a/examples/nexmark.rs +++ b/examples/nexmark.rs @@ -465,11 +465,11 @@ fn main() { tracing_subscriber::fmt::init(); let (config, args) = RuntimeConfig::from_args(); - if args.len() != 2 { + if args.len() != 3 { panic!("Pass the element count as argument"); } - let n: usize = args[0].parse().unwrap(); - let q = &args[1][..]; + let n: usize = args[1].parse().unwrap(); + let q = &args[2][..]; config.spawn_remote_workers(); let env = StreamContext::new(config); diff --git a/src/config.rs b/src/config.rs index 0e57b3cd..38514f37 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,7 +30,7 @@ pub const CONFIG_ENV_VAR: &str = "NOIR_CONFIG"; /// and remote workers). /// /// In a remote execution the current binary is copied using scp to a remote host and then executed -/// using ssh. The configuration of the remote environment should be specified via a YAML +/// using ssh. The configuration of the remote environment should be specified via a TOML /// configuration file. /// /// ## Local environment @@ -47,19 +47,21 @@ pub const CONFIG_ENV_VAR: &str = "NOIR_CONFIG"; /// # use noir_compute::{StreamContext, RuntimeConfig}; /// # use std::fs::File; /// # use std::io::Write; -/// let config = r" -/// hosts: -/// - address: host1 -/// base_port: 9500 -/// num_cores: 16 -/// - address: host2 -/// base_port: 9500 -/// num_cores: 24 -/// "; -/// let mut file = File::create("config.yaml").unwrap(); +/// let config = r#" +/// [[host]] +/// address = "host1" +/// base_port = 9500 +/// num_cores = 16 +/// +/// [[host]] +/// address = "host2" +/// base_port = 9500 +/// num_cores = 24 +/// "#; +/// let mut file = File::create("config.toml").unwrap(); /// file.write_all(config.as_bytes()); /// -/// let config = RuntimeConfig::remote("config.yaml").expect("cannot read config file"); +/// let config = RuntimeConfig::remote("config.toml").expect("cannot read config file"); /// let env = StreamContext::new(config); /// ``` /// @@ -106,6 +108,7 @@ pub struct RemoteConfig { #[serde(skip)] pub host_id: Option, /// The set of remote hosts to use. + #[serde(rename = "host")] pub hosts: Vec, /// If specified some debug information will be stored inside this directory. pub tracing_dir: Option, @@ -233,7 +236,7 @@ impl RuntimeConfig { } else { log::info!("reading config from: {}", config.as_ref().display()); let content = std::fs::read_to_string(config)?; - serde_yaml::from_str(&content)? + toml::from_str(&content)? }; // validate the configuration @@ -274,7 +277,7 @@ impl RuntimeConfig { Ok(config) => { info!("reading remote config from env {}", CONFIG_ENV_VAR); let config: RemoteConfig = - serde_yaml::from_str(&config).expect("Invalid configuration from environment"); + toml::from_str(&config).expect("Invalid configuration from environment"); Some(config) } Err(_) => None, diff --git a/src/lib.rs b/src/lib.rs index bab3efe9..62985be3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,23 +91,23 @@ fn tokenize(s: &str) -> Vec { s.split_whitespace().map(str::to_lowercase).collect() } -// Execute on multiple hosts `cargo run -- -r config.yaml input.txt` +// Execute on multiple hosts `cargo run -- -r config.toml input.txt` ``` ### Remote deployment -```yaml -# config.yaml -hosts: - - address: host1.lan - base_port: 9500 - num_cores: 16 - - address: host2.lan - base_port: 9500 - num_cores: 8 - ssh: - username: noir-compute - key_file: /home/user/.ssh/id_rsa +```toml +# config.toml +[[host]] +address = "host1.lan" +base_port = 9500 +num_cores = 16 + +[[host]] +address = "host2.lan" +base_port = 9500 +num_cores = 24 +ssh = { username = "noir", key_file = "/home/noir/.ssh/id_ed25519" } ``` Refer to the [examples](examples/) directory for an extended set of working examples diff --git a/src/network/topology.rs b/src/network/topology.rs index 35c24b3d..c656c8aa 100644 --- a/src/network/topology.rs +++ b/src/network/topology.rs @@ -631,14 +631,16 @@ mod tests { #[test] fn test_remote_topology() { let mut config = tempfile::NamedTempFile::new().unwrap(); - let config_yaml = "hosts:\n".to_string() - + " - address: 127.0.0.1\n" - + " base_port: 21841\n" - + " num_cores: 1\n" - + " - address: 127.0.0.1\n" - + " base_port: 31258\n" - + " num_cores: 1\n"; - std::io::Write::write_all(&mut config, config_yaml.as_bytes()).unwrap(); + let config_toml = r#"[[host]] +address = "127.0.0.1" +base_port = 21841 +num_cores = 1 +[[host]] +address = "127.0.0.1" +base_port = 31258 +num_cores = 1 +"#; + std::io::Write::write_all(&mut config, config_toml.as_bytes()).unwrap(); let config = RuntimeConfig::remote(config.path()).unwrap(); // s1 [b0, h0, r0] -> r1 [b2, h1, r0] (endpoint 1) type=i32 diff --git a/src/runner.rs b/src/runner.rs index 9236a97b..08500f42 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -378,8 +378,8 @@ fn build_remote_command( binary_path: &Path, perf_path: &Option, ) -> String { - let config_yaml = serde_yaml::to_string(config).unwrap(); - let config_str = shell_escape::escape(config_yaml.into()); + let config_toml = toml::to_string(config).unwrap(); + let config_str = shell_escape::escape(config_toml.into()); let args = std::env::args() .skip(1) .map(|arg| shell_escape::escape(arg.into()))