diff --git a/Makefile b/Makefile index 2f3719a9d..0f0d440cc 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ endif all: manager .PHONY: unittest -unittest: generate lint fmt vet manifests +unittest: generate lint fmt vet manifests metrics-rules-test go test -v -coverprofile cover.out $(SRC_PATHS_TESTS) cd api && go test -v ./... @@ -325,3 +325,22 @@ lint: .PHONY: lint-metrics lint-metrics: ./hack/prom_metric_linter.sh --operator-name="kubevirt" --sub-operator-name="ssp" + +PROMTOOL ?= $(LOCALBIN)/promtool +PROMTOOL_VERSION ?= 2.44.0 + +.PHONY: promtool +promtool: $(PROMTOOL) +$(PROMTOOL): $(LOCALBIN) + test -s $(PROMTOOL) || curl -sSfL "https://github.com/prometheus/prometheus/releases/download/v$(PROMTOOL_VERSION)/prometheus-$(PROMTOOL_VERSION).linux-amd64.tar.gz" | \ + tar xvzf - --directory=$(LOCALBIN) "prometheus-$(PROMTOOL_VERSION).linux-amd64"/promtool --strip-components=1 + +METRIC_RULES_WRITER ?= $(LOCALBIN)/metrics-rules-writer + +.PHONY: build-metric-rules-writer +build-metric-rules-writer: $(LOCALBIN) + go build -o $(METRIC_RULES_WRITER) tools/test-rules-writer/test_rules_writer.go + +.PHONY: metrics-rules-test +metrics-rules-test: build-metric-rules-writer promtool + ./hack/metrics-rules-test.sh $(METRIC_RULES_WRITER) "./pkg/monitoring/rules/rules-tests.yaml" diff --git a/hack/metrics-rules-test.sh b/hack/metrics-rules-test.sh new file mode 100755 index 000000000..04a59aa86 --- /dev/null +++ b/hack/metrics-rules-test.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +readonly PROMTOOL="$(dirname "$0")/../bin/promtool" + +function cleanup() { + local cleanup_dir="${1:?}" + rm -rf "${cleanup_dir}" +} + +function main() { + local prom_spec_dumper="${1:?}" + local tests_file="${2:?}" + local temp_dir + + temp_dir="$(mktemp --tmpdir --directory metrics_test_dir.XXXXX)" + trap "cleanup ${temp_dir}" RETURN EXIT INT + + local rules_file="${temp_dir}/rules.json" + local tests_copy="${temp_dir}/rules-test.yaml" + + "${prom_spec_dumper}" > "${rules_file}" + cp "${tests_file}" "${tests_copy}" + + echo "INFO: Rules file content:" + cat "${rules_file}" + echo + + ${PROMTOOL} check rules "${rules_file}" + ${PROMTOOL} test rules "${tests_copy}" +} + +main "$@" diff --git a/pkg/monitoring/rules/rules-tests.yaml b/pkg/monitoring/rules/rules-tests.yaml new file mode 100644 index 000000000..cb60e2f53 --- /dev/null +++ b/pkg/monitoring/rules/rules-tests.yaml @@ -0,0 +1,26 @@ +# Unit tests for the prometheus rules +rule_files: + - rules.json + +tests: + - interval: "1m" + input_series: + - series: 'up{pod="ssp-operator-12345"}' + values: '0x5 1' + + alert_rule_test: + - eval_time: "5m" + alertname: "SSPDown" + exp_alerts: + - exp_annotations: + summary: "All SSP operator pods are down." + runbook_url: "test-runbook:SSPDown" + exp_labels: + severity: "critical" + operator_health_impact: "critical" + kubernetes_operator_part_of: "kubevirt" + kubernetes_operator_component: "ssp-operator" + + - eval_time: "6m" + alertname: "SSPDown" + exp_alerts: [] diff --git a/tools/test-rules-writer/test_rules_writer.go b/tools/test-rules-writer/test_rules_writer.go new file mode 100644 index 000000000..f571a00df --- /dev/null +++ b/tools/test-rules-writer/test_rules_writer.go @@ -0,0 +1,31 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + + "kubevirt.io/ssp-operator/pkg/monitoring/rules" +) + +func main() { + const runbookTemplate = "test-runbook:%s" + + allRules := append(rules.RecordRules(), rules.AlertRules(runbookTemplate)...) + + spec := promv1.PrometheusRuleSpec{ + Groups: []promv1.RuleGroup{{ + Name: "test.rules", + Rules: allRules, + }}, + } + + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(spec); err != nil { + fmt.Fprintf(os.Stderr, "Error encoding prometheus spec: %v", err) + os.Exit(1) + } +}