Skip to content

Commit

Permalink
Merge pull request #51 from gregcusack/kco-multiple-clients
Browse files Browse the repository at this point in the history 
support multiple clients at large scale
  • Loading branch information
Greg Cusack authored Jan 20, 2024
2 parents be66e75 + 00cc049 commit f5cdd2d
Show file tree
Hide file tree
Showing 4 changed files with 236 additions and 82 deletions.
153 changes: 123 additions & 30 deletions k8s-cluster/src/docker.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
use {
crate::{boxed_error, initialize_globals, ValidatorType, SOLANA_ROOT},
crate::{
boxed_error, initialize_globals, new_spinner_progress_bar, ValidatorType, BUILD, ROCKET,
SOLANA_ROOT,
},
log::*,
rayon::prelude::*,
std::{
error::Error,
fs,
Expand Down Expand Up @@ -32,7 +36,9 @@ impl<'a> DockerConfig<'a> {
}

pub fn build_image(&self, validator_type: &ValidatorType) -> Result<(), Box<dyn Error>> {
match self.create_base_image(validator_type) {
let image_name = format!("{}-{}", validator_type, self.image_config.image_name);
let docker_path = SOLANA_ROOT.join(format!("{}/{}", "docker-build", validator_type));
match self.create_base_image(image_name, docker_path, validator_type, None) {
Ok(res) => {
if res.status.success() {
info!("Successfully created base Image");
Expand All @@ -46,14 +52,39 @@ impl<'a> DockerConfig<'a> {
}
}

pub fn build_client_images(&self, client_count: i32) -> Result<(), Box<dyn Error>> {
for i in 0..client_count {
let image_name = format!("{}-{}-{}", ValidatorType::Client, "image", i);
let docker_path = SOLANA_ROOT.join(format!(
"{}/{}-{}",
"docker-build",
ValidatorType::Client,
i
));
match self.create_base_image(image_name, docker_path, &ValidatorType::Client, Some(i)) {
Ok(res) => {
if res.status.success() {
info!("Successfully created client base Image: {}", i);
} else {
error!("Failed to build client base image: {}", i);
return Err(boxed_error!(String::from_utf8_lossy(&res.stderr)));
}
}
Err(err) => return Err(err),
}
}
Ok(())
}

pub fn create_base_image(
&self,
image_name: String,
docker_path: PathBuf,
validator_type: &ValidatorType,
index: Option<i32>,
) -> Result<Output, Box<dyn Error>> {
let image_name = format!("{}-{}", validator_type, self.image_config.image_name);
let docker_path = SOLANA_ROOT.join(format!("{}/{}", "docker-build", validator_type));

let dockerfile_path = match self.create_dockerfile(validator_type, docker_path, None) {
let dockerfile_path = match self.create_dockerfile(validator_type, docker_path, None, index)
{
Ok(res) => res,
Err(err) => return Err(err),
};
Expand All @@ -66,39 +97,71 @@ impl<'a> DockerConfig<'a> {
// so we result to using std::process::Command
let dockerfile = dockerfile_path.join("Dockerfile");
let context_path = SOLANA_ROOT.display().to_string();

let progress_bar = new_spinner_progress_bar();
progress_bar.set_message(format!(
"{BUILD}Building {} docker image...",
validator_type
));

let command = format!(
"docker build -t {}/{}:{} -f {:?} {}",
self.image_config.registry, image_name, self.image_config.tag, dockerfile, context_path
);
match Command::new("sh")
let output = match Command::new("sh")
.arg("-c")
.arg(&command)
.stdout(Stdio::inherit())
.stderr(Stdio::inherit())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("Failed to execute command")
.wait_with_output()
{
Ok(res) => Ok(res),
Err(err) => Err(Box::new(err)),
}
Err(err) => Err(Box::new(err) as Box<dyn Error>),
};
progress_bar.finish_and_clear();
info!("{} image build complete", validator_type);

output
}

fn insert_client_accounts_if_present(&self) -> String {
if SOLANA_ROOT.join("config-k8s/client-accounts.yml").exists() {
return r#"
COPY --chown=solana:solana ./config-k8s/client-accounts.yml /home/solana
"#
.to_string();
fn insert_client_accounts_if_present(&self, index: Option<i32>) -> String {
match index {
Some(i) => {
if SOLANA_ROOT
.join(format!("config-k8s/bench-tps-{}.yml", i))
.exists()
{
format!(
r#"
COPY --chown=solana:solana ./config-k8s/bench-tps-{}.yml /home/solana/client-accounts.yml
"#,
i
)
} else {
"".to_string()
}
}
None => {
if SOLANA_ROOT.join("config-k8s/client-accounts.yml").exists() {
r#"
COPY --chown=solana:solana ./config-k8s/client-accounts.yml /home/solana
"#
.to_string()
} else {
"".to_string()
}
}
}
"".to_string()
}

pub fn create_dockerfile(
&self,
validator_type: &ValidatorType,
docker_path: PathBuf,
content: Option<&str>,
index: Option<i32>,
) -> Result<PathBuf, Box<dyn std::error::Error>> {
if !(validator_type != &ValidatorType::Bootstrap
|| validator_type != &ValidatorType::Standard
Expand Down Expand Up @@ -157,7 +220,7 @@ WORKDIR /home/solana
let dockerfile = format!(
"{}\n{}",
dockerfile,
self.insert_client_accounts_if_present()
self.insert_client_accounts_if_present(index)
);

debug!("dockerfile: {}", dockerfile);
Expand All @@ -169,21 +232,18 @@ WORKDIR /home/solana
Ok(docker_path)
}

pub fn push_image(&self, validator_type: &ValidatorType) -> Result<(), Box<dyn Error>> {
let image = format!(
"{}/{}-{}:{}",
self.image_config.registry,
validator_type,
self.image_config.image_name,
self.image_config.tag
);

fn push_image(image: String, identifier: &str) -> Result<(), Box<dyn Error + Send>> {
let progress_bar = new_spinner_progress_bar();
progress_bar.set_message(format!(
"{ROCKET}Pushing {} image to registry...",
identifier
));
let command = format!("docker push '{}'", image);
let output = Command::new("sh")
.arg("-c")
.arg(&command)
.stdout(Stdio::inherit())
.stderr(Stdio::inherit())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("Failed to execute command")
.wait_with_output()
Expand All @@ -192,6 +252,39 @@ WORKDIR /home/solana
if !output.status.success() {
return Err(boxed_error!(output.status.to_string()));
}
progress_bar.finish_and_clear();
Ok(())
}

pub fn push_validator_image(
&self,
validator_type: &ValidatorType,
) -> Result<(), Box<dyn Error + Send>> {
info!("Pushing {} image...", validator_type);
let image = format!(
"{}/{}-{}:{}",
self.image_config.registry,
validator_type,
self.image_config.image_name,
self.image_config.tag
);

Self::push_image(image, format!("{}", validator_type).as_str())
}

pub fn push_client_images(&self, num_clients: i32) -> Result<(), Box<dyn Error + Send>> {
info!("Pushing client images...");
(0..num_clients).into_par_iter().try_for_each(|i| {
let image = format!(
"{}/{}-{}-{}:{}",
self.image_config.registry,
ValidatorType::Client,
"image",
i,
self.image_config.tag
);

Self::push_image(image, format!("client-{}", i).as_str())
})
}
}
78 changes: 58 additions & 20 deletions k8s-cluster/src/genesis.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@
#![allow(clippy::arithmetic_side_effects)]

use {
crate::{add_tag_to_name, boxed_error, initialize_globals, ValidatorType, SOLANA_ROOT},
crate::{
add_tag_to_name, boxed_error, initialize_globals, new_spinner_progress_bar, ValidatorType,
SOLANA_ROOT, SUN, WRITING,
},
base64::{engine::general_purpose, Engine as _},
bip39::{Language, Mnemonic, MnemonicType, Seed},
log::*,
Expand Down Expand Up @@ -48,7 +51,6 @@ fn generate_keypair() -> Result<Keypair, ThreadSafeError> {
let mnemonic_type = MnemonicType::for_word_count(DEFAULT_WORD_COUNT).unwrap();
let mnemonic = Mnemonic::new(mnemonic_type, Language::English);
let seed = Seed::new(&mnemonic, &passphrase);
// keypair_from_seed(seed.as_bytes())
keypair_from_seed(seed.as_bytes()).map_err(ThreadSafeError::from)
}

Expand Down Expand Up @@ -247,6 +249,11 @@ impl Genesis {
ValidatorType::Bootstrap => format!("{}-validator", validator_type),
ValidatorType::Standard => "validator".to_string(),
ValidatorType::NonVoting => format!("{}-validator", validator_type),
ValidatorType::Client => {
return Err(boxed_error!(
"Client valdiator_type in generate_accounts not allowed"
))
}
};

let mut filename_prefix_with_optional_tag = filename_prefix;
Expand All @@ -270,7 +277,24 @@ impl Genesis {
Ok(())
}

// TODO: only supports one client right now.
fn create_client_account(
args: &Vec<String>,
executable_path: &PathBuf,
) -> Result<(), Box<dyn Error + Send>> {
let output = Command::new(executable_path)
.args(args)
.output()
.expect("Failed to execute solana-bench-tps");

if !output.status.success() {
return Err(boxed_error!(format!(
"Failed to create client accounts. err: {}",
String::from_utf8_lossy(&output.stderr)
)));
}
Ok(())
}

pub fn create_client_accounts(
&mut self,
number_of_clients: i32,
Expand All @@ -279,7 +303,16 @@ impl Genesis {
build_path: PathBuf,
) -> Result<(), Box<dyn Error>> {
let client_accounts_file = SOLANA_ROOT.join("config-k8s/client-accounts.yml");
for i in 0..number_of_clients {
if bench_tps_args.is_some() {
let list_of_bench_tps_args = bench_tps_args.as_ref().unwrap();
for i in list_of_bench_tps_args.iter() {
info!("bench_tps_arg: {}", i);
}
}

info!("generating {} client accounts...", number_of_clients);
let _ = (0..number_of_clients).into_par_iter().try_for_each(|i| {
info!("client account: {}", i);
let mut args = Vec::new();
let account_path = SOLANA_ROOT.join(format!("config-k8s/bench-tps-{}.yml", i));
args.push("--write-client-keys".to_string());
Expand All @@ -290,27 +323,20 @@ impl Genesis {
if bench_tps_args.is_some() {
let list_of_bench_tps_args = bench_tps_args.as_ref().unwrap();
args.extend(list_of_bench_tps_args.clone()); //can unwrap since we checked is_some()
for i in list_of_bench_tps_args.iter() {
info!("bench_tps_arg: {}", i);
}
}

info!("generating client accounts...");
let executable_path = build_path.join("solana-bench-tps");
let output = Command::new(executable_path)
.args(&args)
.output()
.expect("Failed to execute solana-bench-tps");

if !output.status.success() {
return Err(boxed_error!(format!(
"Failed to create client accounts. err: {}",
String::from_utf8_lossy(&output.stderr)
)));
}

Self::create_client_account(&args, &executable_path)
});

let progress_bar = new_spinner_progress_bar();
progress_bar.set_message(format!("{WRITING}Writing client accounts..."));
for i in 0..number_of_clients {
let account_path = SOLANA_ROOT.join(format!("config-k8s/bench-tps-{}.yml", i));
append_client_accounts_to_file(&account_path, &client_accounts_file)?;
}
progress_bar.finish_and_clear();
info!("client-accounts.yml creation for genesis complete");

// add client accounts file as a primordial account
self.primordial_accounts_files.push(client_accounts_file);
Expand All @@ -335,6 +361,11 @@ impl Genesis {
ValidatorType::Bootstrap => format!("{}/{}.json", filename_prefix, account),
ValidatorType::Standard => format!("{}-{}-{}.json", filename_prefix, account, i),
ValidatorType::NonVoting => format!("{}-{}-{}.json", filename_prefix, account, i),
ValidatorType::Client => {
return Err(boxed_error!(
"Client valdiator_type in generate_account not allowed"
))
}
};

let outfile = self.config_dir.join(&filename);
Expand Down Expand Up @@ -450,18 +481,25 @@ impl Genesis {
debug!("{}", arg);
}

let progress_bar = new_spinner_progress_bar();
progress_bar.set_message(format!("{SUN}Building Genesis..."));

let executable_path = build_path.join("solana-genesis");
let output = Command::new(executable_path)
.args(&args)
.output()
.expect("Failed to execute solana-genesis");

progress_bar.finish_and_clear();

if !output.status.success() {
return Err(boxed_error!(format!(
"Failed to create genesis. err: {}",
String::from_utf8_lossy(&output.stderr)
)));
}
info!("Genesis build complete");

Ok(())
}

Expand Down
Loading

0 comments on commit f5cdd2d

Please sign in to comment.