Add metrics templates

Signed-off-by: Alex Leong <[email protected]>
fluxcd · Sep 15, 2023 · 04a1f2f · 04a1f2f
1 parent efc5880
commit 04a1f2f
Show file tree

Hide file tree

Showing 3 changed files with 192 additions and 11 deletions.
diff --git a/docs/gitbook/tutorials/linkerd-progressive-delivery.md b/docs/gitbook/tutorials/linkerd-progressive-delivery.md
@@ -75,9 +75,65 @@ Create a deployment and a horizontal pod autoscaler:
 kubectl apply -k https://github.com/fluxcd/flagger//kustomize/podinfo?ref=main
 ```
 
-Create a canary custom resource for the podinfo deployment:
+Create a metrics template and canary custom resources for the podinfo deployment:
 
 ```yaml
+---
+apiVersion: flagger.app/v1beta1
+kind: MetricTemplate
+metadata:
+  name: success-rate
+  namespace: test
+spec:
+  provider:
+    type: prometheus
+    address: http://prometheus.linkerd-viz:9090
+  query: |
+    sum(
+      rate(
+        response_total{
+          namespace="{{ namespace }}",
+          deployment=~"{{ target }}",
+          classification!="failure",
+          direction="{{ variables.direction }}"
+        }[{{ interval }}]
+      )
+    ) 
+    / 
+    sum(
+      rate(
+        response_total{
+          namespace="{{ namespace }}",
+          deployment=~"{{ target }}",
+          direction="{{ variables.direction }}"
+        }[{{ interval }}]
+      )
+    ) 
+    * 100
+---
+apiVersion: flagger.app/v1beta1
+kind: MetricTemplate
+metadata:
+  name: latency
+  namespace: test
+spec:
+  provider:
+    type: prometheus
+    address: http://prometheus.linkerd-viz:9090
+  query: |
+    histogram_quantile(
+        0.99,
+        sum(
+            rate(
+                response_latency_ms_bucket{
+                    namespace="{{ namespace }}",
+                    deployment=~"{{ target }}",
+                    direction="{{ variables.direction }}"
+                    }[{{ interval }}]
+                )
+            ) by (le)
+        )
+---
 apiVersion: flagger.app/v1beta1
 kind: Canary
 metadata:
@@ -122,18 +178,28 @@ spec:
     stepWeight: 5
     # Linkerd Prometheus checks
     metrics:
-    - name: request-success-rate
+    - name: success-rate
+      templateRef:
+        name: success-rate
+        namespace: test
       # minimum req success rate (non 5xx responses)
       # percentage (0-100)
       thresholdRange:
         min: 99
       interval: 1m
-    - name: request-duration
+      templateVariables:
+        direction: inbound
+    - name: latency
+      templateRef:
+        name: latency
+        namespace: test
       # maximum req duration P99
       # milliseconds
       thresholdRange:
         max: 500
       interval: 30s
+      templateVariables:
+        direction: inbound
     # testing (optional)
     webhooks:
       - name: acceptance-test

diff --git a/test/linkerd/test-canary.sh b/test/linkerd/test-canary.sh
@@ -6,6 +6,40 @@ set -o errexit
 
 REPO_ROOT=$(git rev-parse --show-toplevel)
 
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: MetricTemplate
+metadata:
+  name: success-rate
+  namespace: linkerd
+spec:
+  provider:
+    type: prometheus
+    address: http://prometheus.linkerd-viz:9090
+  query: |
+    sum(
+      rate(
+        response_total{
+          namespace="{{ namespace }}",
+          deployment=~"{{ target }}",
+          classification!="failure",
+          direction="{{ variables.direction }}"
+        }[{{ interval }}]
+      )
+    ) 
+    / 
+    sum(
+      rate(
+        response_total{
+          namespace="{{ namespace }}",
+          deployment=~"{{ target }}",
+          direction="{{ variables.direction }}"
+        }[{{ interval }}]
+      )
+    ) 
+    * 100
+EOF
+
 cat <<EOF | kubectl apply -f -
 apiVersion: flagger.app/v1beta1
 kind: MetricTemplate
@@ -59,12 +93,14 @@ spec:
     maxWeight: 50
     stepWeight: 10
     metrics:
-    - name: request-success-rate
+    - name: success-rate
+      templateRef:
+        name: success-rate
+        namespace: linkerd
       threshold: 99
       interval: 1m
-    - name: request-duration
-      threshold: 500
-      interval: 30s
+      templateVariables:
+        direction: inbound
     - name: latency
       templateRef:
         name: latency
@@ -218,12 +254,22 @@ spec:
     maxWeight: 50
     stepWeight: 10
     metrics:
-    - name: request-success-rate
+    - name: success-rate
+      templateRef:
+        name: success-rate
+        namespace: linkerd
       threshold: 99
       interval: 1m
-    - name: request-duration
+      templateVariables:
+        direction: inbound
+    - name: latency
+      templateRef:
+        name: latency
+        namespace: linkerd
       threshold: 500
       interval: 30s
+      templateVariables:
+        direction: inbound
     webhooks:
       - name: http-acceptance-test
         type: pre-rollout

diff --git a/test/linkerd/test-steps.sh b/test/linkerd/test-steps.sh
@@ -4,6 +4,65 @@
 
 set -o errexit
 
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: MetricTemplate
+metadata:
+  name: success-rate
+  namespace: linkerd
+spec:
+  provider:
+    type: prometheus
+    address: http://prometheus.linkerd-viz:9090
+  query: |
+    sum(
+      rate(
+        response_total{
+          namespace="{{ namespace }}",
+          deployment=~"{{ target }}",
+          classification!="failure",
+          direction="{{ variables.direction }}"
+        }[{{ interval }}]
+      )
+    ) 
+    / 
+    sum(
+      rate(
+        response_total{
+          namespace="{{ namespace }}",
+          deployment=~"{{ target }}",
+          direction="{{ variables.direction }}"
+        }[{{ interval }}]
+      )
+    ) 
+    * 100
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: MetricTemplate
+metadata:
+  name: latency
+  namespace: linkerd
+spec:
+  provider:
+    type: prometheus
+    address: http://prometheus.linkerd-viz:9090
+  query: |
+    histogram_quantile(
+        0.99,
+        sum(
+            rate(
+                response_latency_ms_bucket{
+                    namespace="{{ namespace }}",
+                    deployment=~"{{ target }}",
+                    direction="{{ variables.direction }}"
+                    }[{{ interval }}]
+                )
+            ) by (le)
+        )
+EOF
+
 cat <<EOF | kubectl apply -f -
 apiVersion: flagger.app/v1beta1
 kind: Canary
@@ -32,12 +91,22 @@ spec:
     maxWeight: 50
     stepWeights: [10, 20, 30, 40, 50, 60]
     metrics:
-    - name: request-success-rate
+    - name: success-rate
+      templateRef:
+        name: success-rate
+        namespace: linkerd
       threshold: 99
       interval: 1m
-    - name: request-duration
+      templateVariables:
+        direction: inbound
+    - name: latency
+      templateRef:
+        name: latency
+        namespace: linkerd
       threshold: 500
       interval: 30s
+      templateVariables:
+        direction: inbound
     webhooks:
       - name: load-test
         url: http://flagger-loadtester.test/