diff --git a/README.md b/README.md index f8654e69de..5b4b2e214f 100644 --- a/README.md +++ b/README.md @@ -346,11 +346,13 @@ After running the above, if you see `Clusterrolebinding "cluster-admin-binding" #### Healthcheck Endpoints -The following healthcheck endpoints are available, some of which are used to determine the result of the aforementioned probes: +The following healthcheck endpoints are available (`self` refers to the telemetry port, while `main` refers to the exposition port): -* `/livez`: Returns a 200 status code if the application is not affected by an outage of the Kubernetes API Server. We recommend to use this as a liveness probe. -* `/metrics`: Returns a 200 status code if the application is able to serve metrics. While this is available for both ports, we recommend to use the telemetry metrics endpoint as a readiness probe. -* `/healthz`: Returns a 200 status code if the application is running. We recommend to use this as a startup probe. +* `/healthz` (exposed on `main`): Returns a 200 status code if the application is running. We recommend to use this for the startup probe. +* `/livez` (exposed on `main`): Returns a 200 status code if the application is not affected by an outage of the Kubernetes API Server. We recommend to using this for the liveness probe. +* `/readyz` (exposed on `self`): Returns a 200 status code if the application is ready to accept requests and expose metrics. We recommend using this for the readiness probe. + +Note that it is discouraged to use the telemetry metrics endpoint for any probe when proxying the exposition data. #### Limited privileges environment diff --git a/README.md.tpl b/README.md.tpl index d5beb3b089..5548b022f5 100644 --- a/README.md.tpl +++ b/README.md.tpl @@ -347,11 +347,13 @@ After running the above, if you see `Clusterrolebinding "cluster-admin-binding" #### Healthcheck Endpoints -The following healthcheck endpoints are available, some of which are used to determine the result of the aforementioned probes: +The following healthcheck endpoints are available (`self` refers to the telemetry port, while `main` refers to the exposition port): -* `/livez`: Returns a 200 status code if the application is not affected by an outage of the Kubernetes API Server. We recommend to use this as a liveness probe. -* `/metrics`: Returns a 200 status code if the application is able to serve metrics. While this is available for both ports, we recommend to use the telemetry metrics endpoint as a readiness probe. -* `/healthz`: Returns a 200 status code if the application is running. We recommend to use this as a startup probe. +* `/healthz` (exposed on `main`): Returns a 200 status code if the application is running. We recommend to use this for the startup probe. +* `/livez` (exposed on `main`): Returns a 200 status code if the application is not affected by an outage of the Kubernetes API Server. We recommend to using this for the liveness probe. +* `/readyz` (exposed on `self`): Returns a 200 status code if the application is ready to accept requests and expose metrics. We recommend using this for the readiness probe. + +Note that it is discouraged to use the telemetry metrics endpoint for any probe when proxying the exposition data. #### Limited privileges environment diff --git a/examples/autosharding/statefulset.yaml b/examples/autosharding/statefulset.yaml index 2b3a8e39b5..a000c2dd93 100644 --- a/examples/autosharding/statefulset.yaml +++ b/examples/autosharding/statefulset.yaml @@ -38,7 +38,7 @@ spec: livenessProbe: httpGet: path: /livez - port: 8080 + port: http-metrics initialDelaySeconds: 5 timeoutSeconds: 5 name: kube-state-metrics @@ -49,8 +49,8 @@ spec: name: telemetry readinessProbe: httpGet: - path: /metrics - port: 8081 + path: /readyz + port: telemetry initialDelaySeconds: 5 timeoutSeconds: 5 securityContext: diff --git a/examples/daemonsetsharding/daemonset.yaml b/examples/daemonsetsharding/daemonset.yaml index e856c673f7..6f5d59a6b1 100644 --- a/examples/daemonsetsharding/daemonset.yaml +++ b/examples/daemonsetsharding/daemonset.yaml @@ -33,7 +33,7 @@ spec: livenessProbe: httpGet: path: /livez - port: 8080 + port: http-metrics initialDelaySeconds: 5 timeoutSeconds: 5 name: kube-state-metrics-shard @@ -44,8 +44,8 @@ spec: name: telemetry readinessProbe: httpGet: - path: /metrics - port: 8081 + path: /readyz + port: telemetry initialDelaySeconds: 5 timeoutSeconds: 5 securityContext: diff --git a/examples/daemonsetsharding/deployment-no-node-pods.yaml b/examples/daemonsetsharding/deployment-no-node-pods.yaml index 8f6e7b4482..268f8fb450 100644 --- a/examples/daemonsetsharding/deployment-no-node-pods.yaml +++ b/examples/daemonsetsharding/deployment-no-node-pods.yaml @@ -28,7 +28,7 @@ spec: livenessProbe: httpGet: path: /livez - port: 8080 + port: http-metrics initialDelaySeconds: 5 timeoutSeconds: 5 name: kube-state-metrics @@ -39,8 +39,8 @@ spec: name: telemetry readinessProbe: httpGet: - path: /metrics - port: 8081 + path: /readyz + port: telemetry initialDelaySeconds: 5 timeoutSeconds: 5 securityContext: diff --git a/examples/daemonsetsharding/deployment.yaml b/examples/daemonsetsharding/deployment.yaml index 3b51b034d6..d54a215b1c 100644 --- a/examples/daemonsetsharding/deployment.yaml +++ b/examples/daemonsetsharding/deployment.yaml @@ -27,7 +27,7 @@ spec: livenessProbe: httpGet: path: /livez - port: 8080 + port: http-metrics initialDelaySeconds: 5 timeoutSeconds: 5 name: kube-state-metrics @@ -38,8 +38,8 @@ spec: name: telemetry readinessProbe: httpGet: - path: /metrics - port: 8081 + path: /readyz + port: telemetry initialDelaySeconds: 5 timeoutSeconds: 5 securityContext: diff --git a/examples/standard/deployment.yaml b/examples/standard/deployment.yaml index ce74e4fbf9..9130ae7497 100644 --- a/examples/standard/deployment.yaml +++ b/examples/standard/deployment.yaml @@ -25,7 +25,7 @@ spec: livenessProbe: httpGet: path: /livez - port: 8080 + port: http-metrics initialDelaySeconds: 5 timeoutSeconds: 5 name: kube-state-metrics @@ -36,8 +36,8 @@ spec: name: telemetry readinessProbe: httpGet: - path: /metrics - port: 8081 + path: /readyz + port: telemetry initialDelaySeconds: 5 timeoutSeconds: 5 securityContext: diff --git a/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet b/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet index 82695482b4..1b2a157dca 100644 --- a/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet +++ b/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet @@ -192,12 +192,12 @@ seccompProfile: { type: 'RuntimeDefault' }, }, livenessProbe: { timeoutSeconds: 5, initialDelaySeconds: 5, httpGet: { - port: 8080, + port: "http-metrics", path: '/livez', } }, readinessProbe: { timeoutSeconds: 5, initialDelaySeconds: 5, httpGet: { - port: 8081, - path: '/metrics', + port: "telemetry", + path: '/readyz', } }, }; diff --git a/pkg/app/server.go b/pkg/app/server.go index 6a7db543e5..253d712fc7 100644 --- a/pkg/app/server.go +++ b/pkg/app/server.go @@ -62,6 +62,7 @@ const ( metricsPath = "/metrics" healthzPath = "/healthz" livezPath = "/livez" + readyzPath = "/readyz" ) // promLogger implements promhttp.Logger @@ -376,6 +377,18 @@ func buildTelemetryServer(registry prometheus.Gatherer) *http.ServeMux { // Add metricsPath mux.Handle(metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorLog: promLogger{}})) + // Add readyzPath + mux.Handle(readyzPath, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + count, err := util.GatherAndCount(registry) + if err != nil || count == 0 { + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(http.StatusText(http.StatusServiceUnavailable))) + return + } + w.WriteHeader(http.StatusOK) + w.Write([]byte(http.StatusText(http.StatusOK))) + })) + // Add index landingConfig := web.LandingConfig{ Name: "kube-state-metrics", @@ -396,6 +409,19 @@ func buildTelemetryServer(registry prometheus.Gatherer) *http.ServeMux { return mux } +func handleClusterDelegationForProber(client kubernetes.Interface, probeType string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + got := client.CoreV1().RESTClient().Get().AbsPath(probeType).Do(context.Background()) + if got.Error() != nil { + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(http.StatusText(http.StatusServiceUnavailable))) + return + } + w.WriteHeader(http.StatusOK) + w.Write([]byte(http.StatusText(http.StatusOK))) + } +} + func buildMetricsServer(m *metricshandler.MetricsHandler, durationObserver prometheus.ObserverVec, client kubernetes.Interface) *http.ServeMux { mux := http.NewServeMux() @@ -410,18 +436,7 @@ func buildMetricsServer(m *metricshandler.MetricsHandler, durationObserver prome mux.Handle(metricsPath, promhttp.InstrumentHandlerDuration(durationObserver, m)) // Add livezPath - mux.Handle(livezPath, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - - // Query the Kube API to make sure we are not affected by a network outage. - got := client.CoreV1().RESTClient().Get().AbsPath("/livez").Do(context.Background()) - if got.Error() != nil { - w.WriteHeader(http.StatusServiceUnavailable) - w.Write([]byte(http.StatusText(http.StatusServiceUnavailable))) - return - } - w.WriteHeader(http.StatusOK) - w.Write([]byte(http.StatusText(http.StatusOK))) - })) + mux.Handle(livezPath, handleClusterDelegationForProber(client, livezPath)) // Add healthzPath mux.HandleFunc(healthzPath, func(w http.ResponseWriter, _ *http.Request) { diff --git a/pkg/util/utils.go b/pkg/util/utils.go index 63e5aba650..14b108f37e 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -21,7 +21,6 @@ import ( "runtime" "strings" - "github.com/prometheus/common/version" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery" @@ -32,6 +31,9 @@ import ( "k8s.io/klog/v2" testUnstructuredMock "k8s.io/sample-controller/pkg/apis/samplecontroller/v1alpha1" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/version" + "k8s.io/kube-state-metrics/v2/pkg/customresource" ) @@ -154,3 +156,19 @@ func GVRFromType(resourceName string, expectedType interface{}) *schema.GroupVer Resource: r, } } + +// GatherAndCount gathers all metrics from the provided Gatherer and counts +// them. It returns the number of metric children in all gathered metric +// families together. +func GatherAndCount(g prometheus.Gatherer) (int, error) { + got, err := g.Gather() + if err != nil { + return 0, fmt.Errorf("gathering metrics failed: %w", err) + } + + result := 0 + for _, mf := range got { + result += len(mf.GetMetric()) + } + return result, nil +}