From 363eed8b82e0aa5cee8fe73ec95498c53d74d49a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Dieguez Date: Mon, 11 Jan 2021 09:02:19 +0100 Subject: [PATCH] Add HTTP endpoint to show overrides (#1324) (#3639) This PR introduces a new endpoint that would retrieve the override configuration, per tenant, that is loaded as part of the runtime configuration Signed-off-by: Mario de Frutos --- CHANGELOG.md | 1 + docs/api/_index.md | 9 +++++++++ docs/configuration/arguments.md | 4 +++- pkg/api/api.go | 8 ++++++++ pkg/api/handlers.go | 12 ++++++++++++ pkg/cortex/modules.go | 2 ++ 6 files changed, 35 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2413daf5da..5cfa138e9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ - Ruler: `-ruler.storage.swift.auth-version`, `-ruler.storage.swift.max-retries`, `-ruler.storage.swift.connect-timeout`, `-ruler.storage.swift.request-timeout`. * [ENHANCEMENT] Disabled in-memory shuffle-sharding subring cache in the store-gateway, ruler and compactor. This should reduce the memory utilisation in these services when shuffle-sharding is enabled, without introducing a significantly increase CPU utilisation. #3601 * [ENHANCEMENT] Shuffle sharding: optimised subring generation used by shuffle sharding. #3601 +* [ENHANCEMENT] New /runtime_config endpoint that returns the defined runtime configuration in YAML format. The returned configuration includes overrides. #3639 * [BUGFIX] Allow `-querier.max-query-lookback` use `y|w|d` suffix like deprecated `-store.max-look-back-period`. #3598 * [BUGFIX] Memberlist: Entry in the ring should now not appear again after using "Forget" feature (unless it's still heartbeating). #3603 * [BUGFIX] Ingester: do not close idle TSDBs while blocks shipping is in progress. #3630 diff --git a/docs/api/_index.md b/docs/api/_index.md index 6bfe8309de..8a453aebf4 100644 --- a/docs/api/_index.md +++ b/docs/api/_index.md @@ -19,6 +19,7 @@ For the sake of clarity, in this document we have grouped API endpoints by servi | --- | ------- | -------- | | [Index page](#index-page) | _All services_ | `GET /` | | [Configuration](#configuration) | _All services_ | `GET /config` | +| [Runtime Configuration](#runtime-configuration) | _All services_ | `GET /runtime_config` | | [Services status](#services-status) | _All services_ | `GET /services` | | [Readiness probe](#readiness-probe) | _All services_ | `GET /ready` | | [Metrics](#metrics) | _All services_ | `GET /metrics` | @@ -123,6 +124,14 @@ GET /config?mode=defaults Displays the configuration using only the default values. +### Runtime Configuration + +``` +GET /runtime_config +``` + +Displays the runtime configuration currently applied to Cortex (in YAML format), including default values. Please be aware that the endpoint will be only available if Cortex is configured with the `-runtime-config.file` option. + ### Services status ``` diff --git a/docs/configuration/arguments.md b/docs/configuration/arguments.md index fb9ebb5c42..71af9cb62a 100644 --- a/docs/configuration/arguments.md +++ b/docs/configuration/arguments.md @@ -286,7 +286,7 @@ Multi KV also reacts on changes done via runtime configuration. It uses this sec ```yaml multi_kv_config: - mirror-enabled: false + mirror_enabled: false primary: memberlist ``` @@ -422,6 +422,8 @@ multi_kv_config: When running Cortex on Kubernetes, store this file in a config map and mount it in each services' containers. When changing the values there is no need to restart the services, unless otherwise specified. +The `/runtime_config` endpoint returns the runtime configuration, including the overrides. + ## Ingester, Distributor & Querier limits. Cortex implements various limits on the requests it can process, in order to prevent a single tenant overwhelming the cluster. There are various default global limits which apply to all tenants which can be set on the command line. These limits can also be overridden on a per-tenant basis by using `overrides` field of runtime configuration file. diff --git a/pkg/api/api.go b/pkg/api/api.go index 328cac260b..412eeff972 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -33,6 +33,7 @@ import ( "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" "github.com/cortexproject/cortex/pkg/util/push" + "github.com/cortexproject/cortex/pkg/util/runtimeconfig" ) type Config struct { @@ -175,6 +176,13 @@ func (a *API) RegisterAPI(httpPathPrefix string, actualCfg interface{}, defaultC a.RegisterRoute("/debug/fgprof", fgprof.Handler(), false, "GET") } +// RegisterRuntimeConfig registers the endpoints associates with the runtime configuration +func (a *API) RegisterRuntimeConfig(runtimeCfgManager *runtimeconfig.Manager) { + a.indexPage.AddLink(SectionAdminEndpoints, "/runtime_config", "Current Runtime Config (incl. Overrides)") + + a.RegisterRoute("/runtime_config", runtimeConfigHandler(runtimeCfgManager), false, "GET") +} + // RegisterDistributor registers the endpoints associated with the distributor. func (a *API) RegisterDistributor(d *distributor.Distributor, pushConfig distributor.Config) { a.RegisterRoute("/api/v1/push", push.Handler(pushConfig, a.sourceIPs, d.Push), true, "POST") diff --git a/pkg/api/handlers.go b/pkg/api/handlers.go index 359dcc7a75..ff893fdd1e 100644 --- a/pkg/api/handlers.go +++ b/pkg/api/handlers.go @@ -32,6 +32,7 @@ import ( "github.com/cortexproject/cortex/pkg/querier" "github.com/cortexproject/cortex/pkg/querier/stats" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/runtimeconfig" ) const ( @@ -221,6 +222,17 @@ func configHandler(actualCfg interface{}, defaultCfg interface{}) http.HandlerFu } } +func runtimeConfigHandler(runtimeCfgManager *runtimeconfig.Manager) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + runtimeConfig := runtimeCfgManager.GetConfig() + if runtimeConfig == nil { + util.WriteTextResponse(w, "runtime config file doesn't exist") + return + } + util.WriteYAMLResponse(w, runtimeConfig) + } +} + // NewQuerierHandler returns a HTTP handler that can be used by the querier service to // either register with the frontend worker query processor or with the external HTTP // server to fulfill the Prometheus query API. diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index ae614079a4..0228b1ba52 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -172,6 +172,7 @@ func (t *Cortex) initRuntimeConfig() (services.Service, error) { serv, err := runtimeconfig.NewRuntimeConfigManager(t.Cfg.RuntimeConfig, prometheus.DefaultRegisterer) t.RuntimeConfig = serv + t.API.RegisterRuntimeConfig(t.RuntimeConfig) return serv, err } @@ -828,6 +829,7 @@ func (t *Cortex) setupModuleManager() error { deps := map[string][]string{ API: {Server}, MemberlistKV: {API}, + RuntimeConfig: {API}, Ring: {API, RuntimeConfig, MemberlistKV}, Overrides: {RuntimeConfig}, Distributor: {DistributorService, API},