From f9372e0ae33ec24e24cc54754eb8410f7e33229f Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 16 Jan 2024 01:09:13 +0000 Subject: [PATCH 1/3] fix client bug and fix metrics bug for multiple deployments --- k8s-cluster/src/main.rs | 21 ++++++++++--------- .../src/scripts/client-startup-script.sh | 2 -- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/k8s-cluster/src/main.rs b/k8s-cluster/src/main.rs index c73c561ddc2116..07cfa9abb3fcce 100644 --- a/k8s-cluster/src/main.rs +++ b/k8s-cluster/src/main.rs @@ -802,15 +802,6 @@ async fn main() { } } - // creates genesis and writes to binary file - match genesis.generate(build_config.build_path()) { - Ok(_) => (), - Err(err) => { - error!("generate genesis error! {}", err); - return; - } - } - // only create client accounts once if client_config.num_clients > 0 && client_config.client_to_run == "bench-tps" { match genesis.create_client_accounts( @@ -826,6 +817,15 @@ async fn main() { } } } + + // creates genesis and writes to binary file + match genesis.generate(build_config.build_path()) { + Ok(_) => (), + Err(err) => { + error!("generate genesis error! {}", err); + return; + } + } } match genesis.generate_accounts( @@ -965,7 +965,8 @@ async fn main() { .value_of("validator_image_name") .expect("Validator image name is required"); - if kub_controller.metrics.is_some() { + // secret create once and use by all pods + if kub_controller.metrics.is_some() && !no_bootstrap { let metrics_secret = match kub_controller.create_metrics_secret() { Ok(secret) => secret, Err(err) => { diff --git a/k8s-cluster/src/scripts/client-startup-script.sh b/k8s-cluster/src/scripts/client-startup-script.sh index 5d13259fa070d0..889f62ee884ade 100755 --- a/k8s-cluster/src/scripts/client-startup-script.sh +++ b/k8s-cluster/src/scripts/client-startup-script.sh @@ -69,11 +69,9 @@ bench-tps) if ${TPU_CLIENT}; then args+=(--use-tpu-client) - # args+=(--url "$BOOTSTRAP_RPC_ADDRESS") args+=(--url "$LOAD_BALANCER_RPC_ADDRESS") elif ${RPC_CLIENT}; then args+=(--use-rpc-client) - # args+=(--url "$BOOTSTRAP_RPC_ADDRESS") args+=(--url "$LOAD_BALANCER_RPC_ADDRESS") else args+=(--entrypoint "$BOOTSTRAP_GOSSIP_ADDRESS") From 874f4ef033859240ff7a88b88aa21511be711f47 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 16 Jan 2024 01:51:36 +0000 Subject: [PATCH 2/3] use --metrics flag instead of the 5 metrics flags for no-bootstrap deployments --- k8s-cluster/README.md | 5 +++++ k8s-cluster/src/kubernetes.rs | 13 +++++++++++++ k8s-cluster/src/main.rs | 11 +++++++++++ 3 files changed, 29 insertions(+) diff --git a/k8s-cluster/README.md b/k8s-cluster/README.md index 98d92d488f2382..dc7a45ba439fed 100644 --- a/k8s-cluster/README.md +++ b/k8s-cluster/README.md @@ -178,6 +178,7 @@ cargo run --bin solana-k8s -- --no-bootstrap ``` + ## Deploying multiple clusters with a client - For a client `bench-tps` test, the client accounts used for loading the validators must be included in the genesis - Therefore, we need to deploy our first cluster as above BUT also need to generate client accounts without actually running the client @@ -244,6 +245,10 @@ cargo run --bin solana-k8s -- ``` ^ note this is not an ideal setup since we have to pass in `--bench-tps-args` for first and last deployment. But `solana-bench-tps` uses `tx-count` to calculate the number of client accounts +## Deploying multiple clusters with a client and metrics +- You only need to pass in all 5 the metrics flags (`--metrics-host`, `--metrics-port`, etc...) on the first deployment where you also deploy the bootstrap validator. Metrics secret is only created once. +- For subsequent deployments, replace these 5 flags with the `--metrics` flag. This will ensure metrics is enabled for subsequent deployments in your cluster. + ## Pod name format ``` kubectl get pods -n diff --git a/k8s-cluster/src/kubernetes.rs b/k8s-cluster/src/kubernetes.rs index ecfe95b2030510..8f682ab1dc9262 100644 --- a/k8s-cluster/src/kubernetes.rs +++ b/k8s-cluster/src/kubernetes.rs @@ -139,6 +139,19 @@ impl Metrics { } } +// default for metrics, initialize it to empty strings +impl Default for Metrics { + fn default() -> Self { + Metrics { + host: Default::default(), + port: Default::default(), + database: Default::default(), + username: Default::default(), + password: Default::default(), + } + } +} + #[derive(Debug, Clone, Copy, PartialEq)] pub enum NodeAffinityType { Equinix, diff --git a/k8s-cluster/src/main.rs b/k8s-cluster/src/main.rs index 07cfa9abb3fcce..4b3e75474bb834 100644 --- a/k8s-cluster/src/main.rs +++ b/k8s-cluster/src/main.rs @@ -429,6 +429,15 @@ fn parse_matches() -> ArgMatches<'static> { .takes_value(true) .help("Metrics Config. Optional: Specify metrics password"), ) + .arg( + Arg::with_name("metrics") + .long("metrics") + .help("Flag for multiple validator deployments in same cluster. Use metrics flags above + for initial deployment (with bootstrap validator). Then you can pass in this flag by itself + along with the --no-bootstrap flag for all the other validator deployments in the same cluster. + All the new validator deployments will use the \"SOLANA_METRICS_CONFIG\" environment variable + created on the first deployment with the bootstrap."), + ) //RPC config .arg( Arg::with_name("number_of_non_voting_validators") @@ -707,6 +716,8 @@ async fn main() { matches.value_of("metrics_username").unwrap().to_string(), matches.value_of("metrics_password").unwrap().to_string(), )) + } else if matches.is_present("metrics") { + Some(Metrics::default()) } else { None }; From 6283655fa64e92ab04ee23149f863bdef1725804 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 16 Jan 2024 01:54:49 +0000 Subject: [PATCH 3/3] derive default --- k8s-cluster/src/kubernetes.rs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/k8s-cluster/src/kubernetes.rs b/k8s-cluster/src/kubernetes.rs index 8f682ab1dc9262..b94729d954e3f3 100644 --- a/k8s-cluster/src/kubernetes.rs +++ b/k8s-cluster/src/kubernetes.rs @@ -106,7 +106,7 @@ pub struct ClientConfig { pub run_client: bool, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct Metrics { pub host: String, pub port: String, @@ -139,19 +139,6 @@ impl Metrics { } } -// default for metrics, initialize it to empty strings -impl Default for Metrics { - fn default() -> Self { - Metrics { - host: Default::default(), - port: Default::default(), - database: Default::default(), - username: Default::default(), - password: Default::default(), - } - } -} - #[derive(Debug, Clone, Copy, PartialEq)] pub enum NodeAffinityType { Equinix,