feat: add startup and liveness probes for bootstrapped nodes (#19)

* feat: add startup and liveness probes for bootstrapped nodes * make each probe configurable * fix check for sidecar * fix test in and expect * leave readiness probe unchanged * Make new probes configurable * fix * fix again * fix one more time
tacoinfra · Sep 5, 2024 · 618c2f1 · 618c2f1
1 parent 2f66918
commit 618c2f1
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 0 deletions.
diff --git a/charts/tezos/templates/_containers.tpl b/charts/tezos/templates/_containers.tpl
@@ -171,6 +171,14 @@
       name: tezos-net
     - containerPort: 9932
       name: metrics
+    {{- if $.node_vals.bootstrapped_startup_probe }}
+  startupProbe:
+    {{- toYaml $.Values.bootstrappedStartupProbe | nindent 4 }}
+    {{- end }}
+    {{- if $.node_vals.bootstrapped_liveness_probe }}
+  livenessProbe:
+    {{- toYaml $.Values.bootstrappedLivenessProbe | nindent 4 }}
+    {{- end  }}
     {{- if or (not (hasKey $.node_vals "bootstrapped_readiness_probe")) $.node_vals.bootstrapped_readiness_probe }}
   readinessProbe:
     httpGet:

diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml
@@ -162,6 +162,12 @@ accounts: {}
 #                                   chain with external bakers, such as a new test chain.
 #                                   Otherwise, the chain may become unreachable externally
 #                                   while waiting for other nodes to come online.
+# - `bootstrapped_startup_probe`: Adds additional startup probe to check checks
+#                                 whether the most recent block is less than 600 seconds old.
+#                                 Defaults to False.
+# - `bootstrapped_liveness_probe`: Adds additional liveness probe to check checks
+#                                 whether the most recent block is less than 600 seconds old.
+#                                 Defaults to False.
 # - `instances`: A list of nodes to fire up, each is a dictionary defining:
 #    - `bake_using_accounts`: List of account names that should be used for baking.
 #    - `authorized_keys`: List of account names that should be used as keys to
@@ -262,6 +268,21 @@ nodes:
             history_mode: rolling
           metrics_addr: ["0.0.0.0:9932"]
 # End nodes
+# default startup probe for bootstrapped nodes
+bootstrappedStartupProbe:
+  httpGet:
+    path: /is_synced
+    port: 31732
+  failureThreshold: 180
+  periodSeconds: 10
+
+# default liveness probe for nodes
+bootstrappedLivenessProbe:
+  httpGet:
+    path: /is_synced
+    port: 31732
+  failureThreshold: 30
+  periodSeconds: 10
 
 # Bakers
 # This deploys "remote bakers" that only bake through RPC.

diff --git a/test/charts/mainnet2.expect.yaml b/test/charts/mainnet2.expect.yaml
@@ -65,6 +65,8 @@ data:
         ]
       },
       "country-town": {
+        "bootstrapped_liveness_probe": true,
+        "bootstrapped_startup_probe": true,
         "env": {
           "all": {
             "key": "specific-pod"
@@ -622,6 +624,18 @@ spec:
               name: tezos-net
             - containerPort: 9932
               name: metrics
+          startupProbe:
+            failureThreshold: 180
+            httpGet:
+              path: /is_synced
+              port: 31732
+            periodSeconds: 10
+          livenessProbe:
+            failureThreshold: 30
+            httpGet:
+              path: /is_synced
+              port: 31732
+            periodSeconds: 10
           readinessProbe:
             httpGet:
               path: /is_synced

diff --git a/test/charts/mainnet2.in.yaml b/test/charts/mainnet2.in.yaml
@@ -29,6 +29,8 @@ nodes:
           history_mode:
             rolling:
               additional_cycles: 5
+    bootstrapped_startup_probe: true
+    bootstrapped_liveness_probe: true
   city-block:
     runs: [tezedge_node, logger, metrics]
     instances: