Add HTTP endpoint to show overrides (cortexproject#1324) (cortexproje…

…ct#3639)

This PR introduces a new endpoint that would retrieve the override
configuration, per tenant, that is loaded as part of the runtime
configuration

Signed-off-by: Mario de Frutos <[email protected]>

CHANGELOG.md

@@ -35,6 +35,7 @@
   - Ruler: `-ruler.storage.swift.auth-version`, `-ruler.storage.swift.max-retries`, `-ruler.storage.swift.connect-timeout`, `-ruler.storage.swift.request-timeout`.
 * [ENHANCEMENT] Disabled in-memory shuffle-sharding subring cache in the store-gateway, ruler and compactor. This should reduce the memory utilisation in these services when shuffle-sharding is enabled, without introducing a significantly increase CPU utilisation. #3601
 * [ENHANCEMENT] Shuffle sharding: optimised subring generation used by shuffle sharding. #3601
+* [ENHANCEMENT] New /runtime_config endpoint that returns the defined runtime configuration in YAML format. The returned configuration includes overrides. #3639
 * [BUGFIX] Allow `-querier.max-query-lookback` use `y|w|d` suffix like deprecated `-store.max-look-back-period`. #3598
 * [BUGFIX] Memberlist: Entry in the ring should now not appear again after using "Forget" feature (unless it's still heartbeating). #3603
 * [BUGFIX] Ingester: do not close idle TSDBs while blocks shipping is in progress. #3630

docs/api/_index.md

@@ -19,6 +19,7 @@ For the sake of clarity, in this document we have grouped API endpoints by servi
 | --- | ------- | -------- |
 | [Index page](#index-page) | _All services_ | `GET /` |
 | [Configuration](#configuration) | _All services_ | `GET /config` |
+| [Runtime Configuration](#runtime-configuration) | _All services_ | `GET /runtime_config` |
 | [Services status](#services-status) | _All services_ | `GET /services` |
 | [Readiness probe](#readiness-probe) | _All services_ | `GET /ready` |
 | [Metrics](#metrics) | _All services_ | `GET /metrics` |
@@ -123,6 +124,14 @@ GET /config?mode=defaults
 
 Displays the configuration using only the default values.
 
+### Runtime Configuration
+
+```
+GET /runtime_config
+```
+
+Displays the runtime configuration currently applied to Cortex (in YAML format), including default values. Please be aware that the endpoint will be only available if Cortex is configured with the `-runtime-config.file` option.
+
 ### Services status
 
 ```

docs/configuration/arguments.md

@@ -286,7 +286,7 @@ Multi KV also reacts on changes done via runtime configuration. It uses this sec
 
 ```yaml
 multi_kv_config:
-    mirror-enabled: false
+    mirror_enabled: false
     primary: memberlist
 ```
 
@@ -422,6 +422,8 @@ multi_kv_config:
 
 When running Cortex on Kubernetes, store this file in a config map and mount it in each services' containers.  When changing the values there is no need to restart the services, unless otherwise specified.
 
+The `/runtime_config` endpoint returns the runtime configuration, including the overrides.
+
 ## Ingester, Distributor & Querier limits.
 
 Cortex implements various limits on the requests it can process, in order to prevent a single tenant overwhelming the cluster.  There are various default global limits which apply to all tenants which can be set on the command line.  These limits can also be overridden on a per-tenant basis by using `overrides` field of runtime configuration file.

pkg/api/api.go

@@ -33,6 +33,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/storegateway"
 	"github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb"
 	"github.com/cortexproject/cortex/pkg/util/push"
+	"github.com/cortexproject/cortex/pkg/util/runtimeconfig"
 )
 
 type Config struct {
@@ -175,6 +176,13 @@ func (a *API) RegisterAPI(httpPathPrefix string, actualCfg interface{}, defaultC
 	a.RegisterRoute("/debug/fgprof", fgprof.Handler(), false, "GET")
 }
 
+// RegisterRuntimeConfig registers the endpoints associates with the runtime configuration
+func (a *API) RegisterRuntimeConfig(runtimeCfgManager *runtimeconfig.Manager) {
+	a.indexPage.AddLink(SectionAdminEndpoints, "/runtime_config", "Current Runtime Config (incl. Overrides)")
+
+	a.RegisterRoute("/runtime_config", runtimeConfigHandler(runtimeCfgManager), false, "GET")
+}
+
 // RegisterDistributor registers the endpoints associated with the distributor.
 func (a *API) RegisterDistributor(d *distributor.Distributor, pushConfig distributor.Config) {
 	a.RegisterRoute("/api/v1/push", push.Handler(pushConfig, a.sourceIPs, d.Push), true, "POST")

pkg/api/handlers.go

@@ -32,6 +32,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/querier"
 	"github.com/cortexproject/cortex/pkg/querier/stats"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/runtimeconfig"
 )
 
 const (
@@ -221,6 +222,17 @@ func configHandler(actualCfg interface{}, defaultCfg interface{}) http.HandlerFu
 	}
 }
 
+func runtimeConfigHandler(runtimeCfgManager *runtimeconfig.Manager) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		runtimeConfig := runtimeCfgManager.GetConfig()
+		if runtimeConfig == nil {
+			util.WriteTextResponse(w, "runtime config file doesn't exist")
+			return
+		}
+		util.WriteYAMLResponse(w, runtimeConfig)
+	}
+}
+
 // NewQuerierHandler returns a HTTP handler that can be used by the querier service to
 // either register with the frontend worker query processor or with the external HTTP
 // server to fulfill the Prometheus query API.

pkg/cortex/modules.go

@@ -172,6 +172,7 @@ func (t *Cortex) initRuntimeConfig() (services.Service, error) {
 
 	serv, err := runtimeconfig.NewRuntimeConfigManager(t.Cfg.RuntimeConfig, prometheus.DefaultRegisterer)
 	t.RuntimeConfig = serv
+	t.API.RegisterRuntimeConfig(t.RuntimeConfig)
 	return serv, err
 }
 
@@ -828,6 +829,7 @@ func (t *Cortex) setupModuleManager() error {
 	deps := map[string][]string{
 		API:                      {Server},
 		MemberlistKV:             {API},
+		RuntimeConfig:            {API},
 		Ring:                     {API, RuntimeConfig, MemberlistKV},
 		Overrides:                {RuntimeConfig},
 		Distributor:              {DistributorService, API},