diff --git a/k8s-cluster/README.md b/k8s-cluster/README.md index 98d92d488f2382..dc7a45ba439fed 100644 --- a/k8s-cluster/README.md +++ b/k8s-cluster/README.md @@ -178,6 +178,7 @@ cargo run --bin solana-k8s -- --no-bootstrap ``` + ## Deploying multiple clusters with a client - For a client `bench-tps` test, the client accounts used for loading the validators must be included in the genesis - Therefore, we need to deploy our first cluster as above BUT also need to generate client accounts without actually running the client @@ -244,6 +245,10 @@ cargo run --bin solana-k8s -- ``` ^ note this is not an ideal setup since we have to pass in `--bench-tps-args` for first and last deployment. But `solana-bench-tps` uses `tx-count` to calculate the number of client accounts +## Deploying multiple clusters with a client and metrics +- You only need to pass in all 5 the metrics flags (`--metrics-host`, `--metrics-port`, etc...) on the first deployment where you also deploy the bootstrap validator. Metrics secret is only created once. +- For subsequent deployments, replace these 5 flags with the `--metrics` flag. This will ensure metrics is enabled for subsequent deployments in your cluster. + ## Pod name format ``` kubectl get pods -n diff --git a/k8s-cluster/src/kubernetes.rs b/k8s-cluster/src/kubernetes.rs index ecfe95b2030510..b94729d954e3f3 100644 --- a/k8s-cluster/src/kubernetes.rs +++ b/k8s-cluster/src/kubernetes.rs @@ -106,7 +106,7 @@ pub struct ClientConfig { pub run_client: bool, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct Metrics { pub host: String, pub port: String, diff --git a/k8s-cluster/src/main.rs b/k8s-cluster/src/main.rs index c73c561ddc2116..4b3e75474bb834 100644 --- a/k8s-cluster/src/main.rs +++ b/k8s-cluster/src/main.rs @@ -429,6 +429,15 @@ fn parse_matches() -> ArgMatches<'static> { .takes_value(true) .help("Metrics Config. Optional: Specify metrics password"), ) + .arg( + Arg::with_name("metrics") + .long("metrics") + .help("Flag for multiple validator deployments in same cluster. Use metrics flags above + for initial deployment (with bootstrap validator). Then you can pass in this flag by itself + along with the --no-bootstrap flag for all the other validator deployments in the same cluster. + All the new validator deployments will use the \"SOLANA_METRICS_CONFIG\" environment variable + created on the first deployment with the bootstrap."), + ) //RPC config .arg( Arg::with_name("number_of_non_voting_validators") @@ -707,6 +716,8 @@ async fn main() { matches.value_of("metrics_username").unwrap().to_string(), matches.value_of("metrics_password").unwrap().to_string(), )) + } else if matches.is_present("metrics") { + Some(Metrics::default()) } else { None }; @@ -802,15 +813,6 @@ async fn main() { } } - // creates genesis and writes to binary file - match genesis.generate(build_config.build_path()) { - Ok(_) => (), - Err(err) => { - error!("generate genesis error! {}", err); - return; - } - } - // only create client accounts once if client_config.num_clients > 0 && client_config.client_to_run == "bench-tps" { match genesis.create_client_accounts( @@ -826,6 +828,15 @@ async fn main() { } } } + + // creates genesis and writes to binary file + match genesis.generate(build_config.build_path()) { + Ok(_) => (), + Err(err) => { + error!("generate genesis error! {}", err); + return; + } + } } match genesis.generate_accounts( @@ -965,7 +976,8 @@ async fn main() { .value_of("validator_image_name") .expect("Validator image name is required"); - if kub_controller.metrics.is_some() { + // secret create once and use by all pods + if kub_controller.metrics.is_some() && !no_bootstrap { let metrics_secret = match kub_controller.create_metrics_secret() { Ok(secret) => secret, Err(err) => { diff --git a/k8s-cluster/src/scripts/client-startup-script.sh b/k8s-cluster/src/scripts/client-startup-script.sh index 5d13259fa070d0..889f62ee884ade 100755 --- a/k8s-cluster/src/scripts/client-startup-script.sh +++ b/k8s-cluster/src/scripts/client-startup-script.sh @@ -69,11 +69,9 @@ bench-tps) if ${TPU_CLIENT}; then args+=(--use-tpu-client) - # args+=(--url "$BOOTSTRAP_RPC_ADDRESS") args+=(--url "$LOAD_BALANCER_RPC_ADDRESS") elif ${RPC_CLIENT}; then args+=(--use-rpc-client) - # args+=(--url "$BOOTSTRAP_RPC_ADDRESS") args+=(--url "$LOAD_BALANCER_RPC_ADDRESS") else args+=(--entrypoint "$BOOTSTRAP_GOSSIP_ADDRESS")