diff --git a/CHANGELOG.md b/CHANGELOG.md index 88620bafd3..2413daf5da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,9 @@ * [ENHANCEMENT] Compactor: tenants marked for deletion will now be fully cleaned up after some delay since deletion of last block. Cleanup includes removal of remaining marker files (including tenant deletion mark file) and files under `debug/metas`. #3613 * [ENHANCEMENT] Compactor: retry compaction of a single tenant on failure instead of re-running compaction for all tenants. #3627 * [ENHANCEMENT] Querier: Implement result caching for tenant query federation. #3640 +* [ENHANCEMENT] API: Add a `mode` query parameter for the config endpoint: #3645 + * `/config?mode=diff`: Shows the YAML configuration with all values that differ from the defaults. + * `/config?mode=defaults`: Shows the YAML configuration with all the default values. * [ENHANCEMENT] OpenStack Swift: added the following config options to OpenStack Swift backend client: #3660 - Chunks storage: `-swift.auth-version`, `-swift.max-retries`, `-swift.connect-timeout`, `-swift.request-timeout`. - Blocks storage: ` -blocks-storage.swift.auth-version`, ` -blocks-storage.swift.max-retries`, ` -blocks-storage.swift.connect-timeout`, ` -blocks-storage.swift.request-timeout`. diff --git a/docs/api/_index.md b/docs/api/_index.md index 96448a72af..6bfe8309de 100644 --- a/docs/api/_index.md +++ b/docs/api/_index.md @@ -109,6 +109,20 @@ GET /config Displays the configuration currently applied to Cortex (in YAML format), including default values and settings via CLI flags. Sensitive data is masked. Please be aware that the exported configuration **doesn't include the per-tenant overrides**. +#### Different modes + +``` +GET /config?mode=diff +``` + +Displays the configuration currently applied to Cortex (in YAML format) as before, but containing only the values that differ from the default values. + +``` +GET /config?mode=defaults +``` + +Displays the configuration using only the default values. + ### Services status ``` diff --git a/pkg/api/api.go b/pkg/api/api.go index eecd83f8ab..328cac260b 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -166,10 +166,11 @@ func (a *API) RegisterAlertmanager(am *alertmanager.MultitenantAlertmanager, tar } // RegisterAPI registers the standard endpoints associated with a running Cortex. -func (a *API) RegisterAPI(httpPathPrefix string, cfg interface{}) { - a.indexPage.AddLink(SectionAdminEndpoints, "/config", "Current Config") +func (a *API) RegisterAPI(httpPathPrefix string, actualCfg interface{}, defaultCfg interface{}) { + a.indexPage.AddLink(SectionAdminEndpoints, "/config", "Current Config (including the default values)") + a.indexPage.AddLink(SectionAdminEndpoints, "/config?mode=diff", "Current Config (show only values that differ from the defaults)") - a.RegisterRoute("/config", configHandler(cfg), false, "GET") + a.RegisterRoute("/config", configHandler(actualCfg, defaultCfg), false, "GET") a.RegisterRoute("/", indexHandler(httpPathPrefix, a.indexPage), false, "GET") a.RegisterRoute("/debug/fgprof", fgprof.Handler(), false, "GET") } diff --git a/pkg/api/handlers.go b/pkg/api/handlers.go index 9afffcc873..359dcc7a75 100644 --- a/pkg/api/handlers.go +++ b/pkg/api/handlers.go @@ -2,14 +2,15 @@ package api import ( "context" + "fmt" "html/template" "net/http" "path" + "reflect" "regexp" "sync" "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" "github.com/gorilla/mux" "github.com/opentracing-contrib/go-stdlib/nethttp" "github.com/opentracing/opentracing-go" @@ -115,20 +116,109 @@ func indexHandler(httpPathPrefix string, content *IndexPageContent) http.Handler } } -func configHandler(cfg interface{}) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - out, err := yaml.Marshal(cfg) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return +func yamlMarshalUnmarshal(in interface{}) (map[interface{}]interface{}, error) { + yamlBytes, err := yaml.Marshal(in) + if err != nil { + return nil, err + } + + object := make(map[interface{}]interface{}) + if err := yaml.Unmarshal(yamlBytes, object); err != nil { + return nil, err + } + + return object, nil +} + +func diffConfig(defaultConfig, actualConfig map[interface{}]interface{}) (map[interface{}]interface{}, error) { + output := make(map[interface{}]interface{}) + + for key, value := range actualConfig { + + defaultValue, ok := defaultConfig[key] + if !ok { + output[key] = value + continue } - w.Header().Set("Content-Type", "text/yaml") - w.WriteHeader(http.StatusOK) - if _, err := w.Write(out); err != nil { - level.Error(util.Logger).Log("msg", "error writing response", "err", err) + switch v := value.(type) { + case int: + defaultV, ok := defaultValue.(int) + if !ok || defaultV != v { + output[key] = v + } + case string: + defaultV, ok := defaultValue.(string) + if !ok || defaultV != v { + output[key] = v + } + case bool: + defaultV, ok := defaultValue.(bool) + if !ok || defaultV != v { + output[key] = v + } + case []interface{}: + defaultV, ok := defaultValue.([]interface{}) + if !ok || !reflect.DeepEqual(defaultV, v) { + output[key] = v + } + case float64: + defaultV, ok := defaultValue.(float64) + if !ok || !reflect.DeepEqual(defaultV, v) { + output[key] = v + } + case map[interface{}]interface{}: + defaultV, ok := defaultValue.(map[interface{}]interface{}) + if !ok { + output[key] = value + } + diff, err := diffConfig(defaultV, v) + if err != nil { + return nil, err + } + if len(diff) > 0 { + output[key] = diff + } + default: + return nil, fmt.Errorf("unsupported type %T", v) } } + + return output, nil +} + +func configHandler(actualCfg interface{}, defaultCfg interface{}) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var output interface{} + switch r.URL.Query().Get("mode") { + case "diff": + defaultCfgObj, err := yamlMarshalUnmarshal(defaultCfg) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + actualCfgObj, err := yamlMarshalUnmarshal(actualCfg) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + diff, err := diffConfig(defaultCfgObj, actualCfgObj) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + output = diff + + case "defaults": + output = defaultCfg + default: + output = actualCfg + } + + util.WriteYAMLResponse(w, output) + } } // NewQuerierHandler returns a HTTP handler that can be used by the querier service to diff --git a/pkg/api/handlers_test.go b/pkg/api/handlers_test.go index 36a8e42c22..108113437f 100644 --- a/pkg/api/handlers_test.go +++ b/pkg/api/handlers_test.go @@ -1,10 +1,12 @@ package api import ( + "io/ioutil" "net/http/httptest" "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -53,3 +55,110 @@ func TestIndexPageContent(t *testing.T) { require.True(t, strings.Contains(resp.Body.String(), "/shutdown")) require.False(t, strings.Contains(resp.Body.String(), "/compactor/ring")) } + +type diffConfigMock struct { + MyInt int `yaml:"my_int"` + MyFloat float64 `yaml:"my_float"` + MySlice []string `yaml:"my_slice"` + IgnoredField func() error `yaml:"-"` + MyNestedStruct struct { + MyString string `yaml:"my_string"` + MyBool bool `yaml:"my_bool"` + MyEmptyStruct struct{} `yaml:"my_empty_struct"` + } `yaml:"my_nested_struct"` +} + +func newDefaultDiffConfigMock() *diffConfigMock { + c := &diffConfigMock{ + MyInt: 666, + MyFloat: 6.66, + MySlice: []string{"value1", "value2"}, + IgnoredField: func() error { return nil }, + } + c.MyNestedStruct.MyString = "string1" + return c +} + +func TestConfigDiffHandler(t *testing.T) { + for _, tc := range []struct { + name string + expectedStatusCode int + expectedBody string + actualConfig func() interface{} + }{ + { + name: "no config parameters overridden", + expectedStatusCode: 200, + expectedBody: "{}\n", + }, + { + name: "slice changed", + actualConfig: func() interface{} { + c := newDefaultDiffConfigMock() + c.MySlice = append(c.MySlice, "value3") + return c + }, + expectedStatusCode: 200, + expectedBody: "my_slice:\n" + + "- value1\n" + + "- value2\n" + + "- value3\n", + }, + { + name: "string in nested struct changed", + actualConfig: func() interface{} { + c := newDefaultDiffConfigMock() + c.MyNestedStruct.MyString = "string2" + return c + }, + expectedStatusCode: 200, + expectedBody: "my_nested_struct:\n" + + " my_string: string2\n", + }, + { + name: "bool in nested struct changed", + actualConfig: func() interface{} { + c := newDefaultDiffConfigMock() + c.MyNestedStruct.MyBool = true + return c + }, + expectedStatusCode: 200, + expectedBody: "my_nested_struct:\n" + + " my_bool: true\n", + }, + { + name: "test invalid input", + actualConfig: func() interface{} { + c := "x" + return &c + }, + expectedStatusCode: 500, + expectedBody: "yaml: unmarshal errors:\n" + + " line 1: cannot unmarshal !!str `x` into map[interface {}]interface {}\n", + }, + } { + defaultCfg := newDefaultDiffConfigMock() + t.Run(tc.name, func(t *testing.T) { + + var actualCfg interface{} + if tc.actualConfig != nil { + actualCfg = tc.actualConfig() + } else { + actualCfg = newDefaultDiffConfigMock() + } + + req := httptest.NewRequest("GET", "http://test.com/config?mode=diff", nil) + w := httptest.NewRecorder() + + h := configHandler(actualCfg, defaultCfg) + h(w, req) + resp := w.Result() + assert.Equal(t, tc.expectedStatusCode, resp.StatusCode) + + body, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err) + assert.Equal(t, tc.expectedBody, string(body)) + }) + } + +} diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index dac5350f92..ae614079a4 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -1,6 +1,7 @@ package cortex import ( + "flag" "fmt" "os" "time" @@ -80,6 +81,13 @@ const ( All string = "all" ) +func newDefaultConfig() *Config { + defaultConfig := &Config{} + defaultFS := flag.NewFlagSet("", flag.PanicOnError) + defaultConfig.RegisterFlags(defaultFS) + return defaultConfig +} + func (t *Cortex) initAPI() (services.Service, error) { t.Cfg.API.ServerPrefix = t.Cfg.Server.PathPrefix t.Cfg.API.LegacyHTTPPrefix = t.Cfg.HTTPPrefix @@ -90,8 +98,7 @@ func (t *Cortex) initAPI() (services.Service, error) { } t.API = a - - t.API.RegisterAPI(t.Cfg.Server.PathPrefix, t.Cfg) + t.API.RegisterAPI(t.Cfg.Server.PathPrefix, t.Cfg, newDefaultConfig()) return nil, nil } diff --git a/pkg/cortex/modules_test.go b/pkg/cortex/modules_test.go new file mode 100644 index 0000000000..57e1fc1369 --- /dev/null +++ b/pkg/cortex/modules_test.go @@ -0,0 +1,101 @@ +package cortex + +import ( + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/weaveworks/common/server" +) + +func changeTargetConfig(c *Config) { + c.Target = []string{"all", "ruler"} +} + +func TestAPIConfig(t *testing.T) { + actualCfg := newDefaultConfig() + + cortex := &Cortex{ + Server: &server.Server{}, + } + + for _, tc := range []struct { + name string + path string + actualCfg func(*Config) + expectedStatusCode int + expectedBody func(*testing.T, string) + }{ + { + name: "running with default config", + path: "/config", + expectedStatusCode: 200, + }, + { + name: "defaults with default config", + path: "/config?mode=defaults", + expectedStatusCode: 200, + }, + { + name: "diff with default config", + path: "/config?mode=diff", + expectedStatusCode: 200, + expectedBody: func(t *testing.T, body string) { + assert.Equal(t, "{}\n", body) + }, + }, + { + name: "running with changed target config", + path: "/config", + actualCfg: changeTargetConfig, + expectedStatusCode: 200, + expectedBody: func(t *testing.T, body string) { + assert.Contains(t, body, "target: all,ruler\n") + }, + }, + { + name: "defaults with changed target config", + path: "/config?mode=defaults", + actualCfg: changeTargetConfig, + expectedStatusCode: 200, + expectedBody: func(t *testing.T, body string) { + assert.Contains(t, body, "target: all\n") + }, + }, + { + name: "diff with changed target config", + path: "/config?mode=diff", + actualCfg: changeTargetConfig, + expectedStatusCode: 200, + expectedBody: func(t *testing.T, body string) { + assert.Equal(t, "target: all,ruler\n", body) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cortex.Server.HTTP = mux.NewRouter() + + cortex.Cfg = *actualCfg + if tc.actualCfg != nil { + tc.actualCfg(&cortex.Cfg) + } + + _, err := cortex.initAPI() + require.NoError(t, err) + + req := httptest.NewRequest("GET", tc.path, nil) + resp := httptest.NewRecorder() + + cortex.Server.HTTP.ServeHTTP(resp, req) + + assert.Equal(t, tc.expectedStatusCode, resp.Code) + + if tc.expectedBody != nil { + tc.expectedBody(t, resp.Body.String()) + } + }) + } + +} diff --git a/pkg/util/http.go b/pkg/util/http.go index f73f530ea0..0af71f1bd4 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -15,6 +15,7 @@ import ( "github.com/golang/snappy" "github.com/opentracing/opentracing-go" otlog "github.com/opentracing/opentracing-go/log" + "gopkg.in/yaml.v2" ) // WriteJSONResponse writes some JSON as a HTTP response. @@ -33,6 +34,24 @@ func WriteJSONResponse(w http.ResponseWriter, v interface{}) { _, _ = w.Write(data) } +// WriteYAMLResponse writes some YAML as a HTTP response. +func WriteYAMLResponse(w http.ResponseWriter, v interface{}) { + // There is not standardised content-type for YAML, text/plain ensures the + // YAML is displayed in the browser instead of offered as a download + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + + data, err := yaml.Marshal(v) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // We ignore errors here, because we cannot do anything about them. + // Write will trigger sending Status code, so we cannot send a different status code afterwards. + // Also this isn't internal error, but error communicating with client. + _, _ = w.Write(data) +} + // Sends message as text/plain response with 200 status code. func WriteTextResponse(w http.ResponseWriter, message string) { w.Header().Set("Content-Type", "text/plain")