Merge pull request #49 from gregcusack/kco-fix-client-bug

fix client bug and fix metrics bug for multiple deployments

k8s-cluster/README.md

@@ -178,6 +178,7 @@ cargo run --bin solana-k8s --
     --no-bootstrap
 ```
 
+
 ## Deploying multiple clusters with a client
 - For a client `bench-tps` test, the client accounts used for loading the validators must be included in the genesis
 - Therefore, we need to deploy our first cluster as above BUT also need to generate client accounts without actually running the client
@@ -244,6 +245,10 @@ cargo run --bin solana-k8s --
 ```
 ^ note this is not an ideal setup since we have to pass in `--bench-tps-args` for first and last deployment. But `solana-bench-tps` uses `tx-count` to calculate the number of client accounts
 
+## Deploying multiple clusters with a client and metrics
+- You only need to pass in all 5 the metrics flags (`--metrics-host`, `--metrics-port`, etc...) on the first deployment where you also deploy the bootstrap validator. Metrics secret is only created once.
+- For subsequent deployments, replace these 5 flags with the `--metrics` flag. This will ensure metrics is enabled for subsequent deployments in your cluster.
+
 ## Pod name format
 ```
 kubectl get pods -n <namespace>

k8s-cluster/src/kubernetes.rs

@@ -106,7 +106,7 @@ pub struct ClientConfig {
     pub run_client: bool,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default)]
 pub struct Metrics {
     pub host: String,
     pub port: String,

k8s-cluster/src/main.rs

@@ -429,6 +429,15 @@ fn parse_matches() -> ArgMatches<'static> {
                 .takes_value(true)
                 .help("Metrics Config. Optional: Specify metrics password"),
         )
+        .arg(
+            Arg::with_name("metrics")
+                .long("metrics")
+                .help("Flag for multiple validator deployments in same cluster. Use metrics flags above
+                for initial deployment (with bootstrap validator). Then you can pass in this flag by itself
+                along with the --no-bootstrap flag for all the other validator deployments in the same cluster.
+                All the new validator deployments will use the \"SOLANA_METRICS_CONFIG\" environment variable
+                created on the first deployment with the bootstrap."),
+        )
         //RPC config
         .arg(
             Arg::with_name("number_of_non_voting_validators")
@@ -707,6 +716,8 @@ async fn main() {
             matches.value_of("metrics_username").unwrap().to_string(),
             matches.value_of("metrics_password").unwrap().to_string(),
         ))
+    } else if matches.is_present("metrics") {
+        Some(Metrics::default())
     } else {
         None
     };
@@ -802,15 +813,6 @@ async fn main() {
                 }
             }
 
-            // creates genesis and writes to binary file
-            match genesis.generate(build_config.build_path()) {
-                Ok(_) => (),
-                Err(err) => {
-                    error!("generate genesis error! {}", err);
-                    return;
-                }
-            }
-
             // only create client accounts once
             if client_config.num_clients > 0 && client_config.client_to_run == "bench-tps" {
                 match genesis.create_client_accounts(
@@ -826,6 +828,15 @@ async fn main() {
                     }
                 }
             }
+
+            // creates genesis and writes to binary file
+            match genesis.generate(build_config.build_path()) {
+                Ok(_) => (),
+                Err(err) => {
+                    error!("generate genesis error! {}", err);
+                    return;
+                }
+            }
         }
 
         match genesis.generate_accounts(
@@ -965,7 +976,8 @@ async fn main() {
         .value_of("validator_image_name")
         .expect("Validator image name is required");
 
-    if kub_controller.metrics.is_some() {
+    // secret create once and use by all pods
+    if kub_controller.metrics.is_some() && !no_bootstrap {
         let metrics_secret = match kub_controller.create_metrics_secret() {
             Ok(secret) => secret,
             Err(err) => {

k8s-cluster/src/scripts/client-startup-script.sh

@@ -69,11 +69,9 @@ bench-tps)
 
   if ${TPU_CLIENT}; then
     args+=(--use-tpu-client)
-    # args+=(--url "$BOOTSTRAP_RPC_ADDRESS")
     args+=(--url "$LOAD_BALANCER_RPC_ADDRESS")
   elif ${RPC_CLIENT}; then
     args+=(--use-rpc-client)
-    # args+=(--url "$BOOTSTRAP_RPC_ADDRESS")
     args+=(--url "$LOAD_BALANCER_RPC_ADDRESS")
   else
     args+=(--entrypoint "$BOOTSTRAP_GOSSIP_ADDRESS")