Add non-blocking opsrecipe validation (#1097)

* Add non-blocking opsrecipe validation

* Add non-blocking opsrecipe validation

* better bash

* Add intranet/handbook merging as in intranet

* fix syntax error

* Update Makefile.custom.mk

* Update test/hack/bin/check-opsrecipes.sh

* Update test/hack/bin/check-opsrecipes.sh

* Update test/hack/bin/check-opsrecipes.sh

Co-authored-by: Hervé Nicol <[email protected]>

* Update test/hack/bin/check-opsrecipes.sh

Co-authored-by: Hervé Nicol <[email protected]>

---------

Co-authored-by: Jonas Zeiger <[email protected]>
Co-authored-by: Hervé Nicol <[email protected]>

.github/workflows/alert_tests.yaml

@@ -20,3 +20,19 @@ jobs:
           fetch-depth: "0"
       - name: run inhibition tests
         run: make test-inhibitions
+  opsrecipe-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
+      - name: Clone private opsrecipe repo
+        run: |
+          git clone --depth 1 --single-branch -b main -q https://oauth2:${{ secrets.TAYLORBOT_GITHUB_ACTION }}@github.com/giantswarm/giantswarm.git
+      - name: Clone public opsrecipe repo
+        run: |
+          git clone --depth 1 --single-branch -b main -q https://oauth2:${{ secrets.TAYLORBOT_GITHUB_ACTION }}@github.com/giantswarm/handbook.git
+      - name: run opsrecipe tests
+        env:
+          OPSRECIPES_DIR: ./giantswarm
+        run: make test-ci-opsrecipes

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add non-blocking opsrecipe validation.
+
 ### Fixed
 
 - Fix `WorkloadClusterMasterMemoryUsageTooHigh` opsrecipe.

Makefile.custom.mk

@@ -6,16 +6,11 @@ clean-dry-run: ## dry run for `make clean` - print all untracked files
 clean: ## Clean the git work dir and remove all untracked files
 	# clean stage
 	git clean -xdf -- test/hack/bin test/hack/output test/hack/checkLabels
-	git checkout -- helm/prometheus-rules/Chart.yaml
-	git checkout -- helm/prometheus-rules/values.yaml
 
 ##@ Testing
 
 .PHONY: test
-test: install-tools template-chart test-rules test-inhibitions restore-chart
-
-test-rules: install-tools template-chart ## run unit tests for alerting rules
-	bash test/hack/bin/verify-rules.sh "$(test_filter)"
+test: install-tools template-chart test-rules test-inhibitions test-opsrecipes ## Run all tests
 
 install-tools:
 	./test/hack/bin/fetch-tools.sh
@@ -24,16 +19,15 @@ template-chart: install-tools ## prepare the helm chart
 	test/hack/bin/architect helm template --dir helm/prometheus-rules --dry-run
 	bash ./test/hack/bin/template-chart.sh
 
+test-rules: install-tools template-chart ## run unit tests for alerting rules
+	bash test/hack/bin/verify-rules.sh "$(test_filter)"
+
 test-inhibitions: install-tools template-chart ## test whether inhibition labels are well defined
-	./test/hack/bin/get-inhibition.sh
+	bash test/hack/bin/get-inhibition.sh
 	cd test/hack/checkLabels; go run main.go
 
 test-opsrecipes: install-tools template-chart ## Check if opsrecipes are valid
-	./test/hack/bin/check-opsrecipes.sh
-
-restore-chart:
-	@## Revert Chart version
-	@yq e -i '.version = "[[ .Version ]]"' helm/prometheus-rules/Chart.yaml
-	@yq e -i '.appVersion = "[[ .AppVersion ]]"' helm/prometheus-rules/Chart.yaml
-	@yq e -i '.project.branch = "[[ .Branch ]]"' helm/prometheus-rules/values.yaml
-	@yq e -i '.project.commit = "[[ .SHA ]]"' helm/prometheus-rules/values.yaml
+	bash test/hack/bin/check-opsrecipes.sh
+
+test-ci-opsrecipes: install-tools template-chart ## Check if opsrecipes are valid in CI
+	test/hack/bin/check-opsrecipes.sh --ci

test/conf/promtool_ignore

@@ -64,4 +64,5 @@ templates/recording-rules/gs-managed-app-deployment-status.rules.yml
 templates/recording-rules/kubernetes-mixins.rules.yml
 templates/recording-rules/service-level.rules.yml
 templates/recording-rules/mimir-mixins.rules.yml
+templates/recording-rules/tempo-mixins.rules.yml
 templates/recording-rules/loki-mixins.rules.yml

test/hack/bin/check-opsrecipes.sh

@@ -6,6 +6,7 @@ RULES_FILES=(./test/hack/output/*/*/prometheus-rules/templates/alerting-rules/*)
 #RULES_FILES=(./test/hack/output/*/prometheus-rules/templates/alerting-rules/up*)
 
 DEBUG_MODE=false
+
 CHECK_EXTRADATA_ERRORS=true
 CHECK_NORECIPE_ERRORS=true
 CHECK_UNEXISTINGRECIPE_ERRORS=true
@@ -27,15 +28,40 @@ isInArray () {
     return 1
 }
 
+# merge_docs () merges a Hugo docs hierarchy from a source directory (first arg) into a destination directory (second arg).
+merge_docs() {
+    if [[ ! -d "$1/content/docs/." ]] ; then
+        echo "Source Hugo base directory not specified or invalid (must contain content/docs)!" >&2
+    fi
+    if [[ ! -d "$2/content/docs/." ]] ; then
+        echo "Destination Hugo base directory not specified or invalid (must contain content/docs)!" >&2
+    fi
+    find "$1/content/docs" -mindepth 1 -maxdepth 1 -type d | xargs cp -v -x -a -r -u -t "$2/content/docs/." | \
+        grep -o -P "(?<= -> ').*\.md" | \
+        xargs sed -s -i'' '0,/^---.*$/s//---\nsourceOrigin: handbook/'
+}
 
 listOpsRecipes () {
-    # find list of opsrecipes from git repo
-    tmpDir="$(mktemp -d)"
-    git clone --depth 1 --single-branch -b main -q [email protected]:giantswarm/giantswarm.git "$tmpDir"
+    local runInCi="$1" && shift
+    privateOpsrecipesParentDirectory="./giantswarm"
+    privateOpsrecipesHandbookParentDirectory="./handbook"
+    # CI clones git dependencies, but if we run it locally we have to do it ourselves
+    if [[ "$runInCi" == false ]]; then
+        tmpDir="$(mktemp -d)"
+        tmpDirHandbook="$(mktemp -d)"
+        git clone --depth 1 --single-branch -b main -q [email protected]:giantswarm/giantswarm.git "$tmpDir"
+        git clone --depth=1 --single-branch -b main -q [email protected]:giantswarm/handbook.git "$tmpDirHandbook"
+        privateOpsrecipesParentDirectory="$tmpDir"
+        privateOpsrecipesHandbookParentDirectory="$tmpDirHandbook"
+    fi
+
+    # perform merge as done by intranet build
+    merge_docs "$privateOpsrecipesHandbookParentDirectory" "$privateOpsrecipesParentDirectory"
+
     # find all ops-recipes ".md" files, and keep only the opsrecipe name (may contain a path, like "rolling-nodes/rolling-nodes")
-    find "$tmpDir"/content/docs/support-and-ops/ops-recipes -type f -name \*.md \
-        | sed -n 's_'"$tmpDir"'/content/docs/support-and-ops/ops-recipes/\(.*\).md_\1_p'
-    rm -rf "$tmpDir"
+    find "$privateOpsrecipesParentDirectory"/content/docs/support-and-ops/ops-recipes -type f -name \*.md \
+        | sed -n 's_'"$privateOpsrecipesParentDirectory"'/content/docs/support-and-ops/ops-recipes/\(.*\).md_\1_p'
+    rm -rf "$privateOpsrecipesParentDirectory"
 
     # Add extra opsrecipes
     # These ones are defined as aliases of `deployment-not-satisfied`:
@@ -46,6 +72,14 @@ listOpsRecipes () {
 
 
 main() {
+    local -a runInCi=false
+    for arg in "$@"; do
+        shift
+        case "$arg" in
+            '--ci')  runInCi=true   ;;
+        esac
+    done
+
     local -a checkedRules=()
     local -a opsRecipes=()
     local -a E_extradata=()
@@ -57,7 +91,7 @@ main() {
     ########################
 
     # Retrieve list of opsrecipes
-    mapfile -t opsRecipes < <(listOpsRecipes)
+    mapfile -t opsRecipes < <(listOpsRecipes "$runInCi")
 
     if [[ "$DEBUG_MODE" != "false" ]]; then
         echo "List of opsrecipe:"

test/hack/bin/template-chart.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 main() {
-
   local GIT_WORKDIR
   GIT_WORKDIR="$(git rev-parse --show-toplevel)"