diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a1193fe5f..6d75ed43f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## master / unreleased -* [ENHANCEMENT] Memberlist: add status page with available details about memberlist-based KV store and memberlist cluster. It's also possible to view KV values in Go struct or JSON format, or download for inspection. #3575 +* [ENHANCEMENT] Memberlist: add status page (/memberlist) with available details about memberlist-based KV store and memberlist cluster. It's also possible to view KV values in Go struct or JSON format, or download for inspection. #3575 +* [ENHANCEMENT] Memberlist: client can now keep a size-bounded buffer with sent and received messages and display them in the admin UI (/memberlist) for troubleshooting. #3581 ## 1.6.0-rc.0 in progress diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index ae16e63ef9..9798863c96 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2979,6 +2979,11 @@ The `memberlist_config` configures the Gossip memberlist. # CLI flag: -memberlist.leave-timeout [leave_timeout: | default = 5s] +# How much space to use for keeping received and sent messages in memory for +# troubleshooting (two buffers). 0 to disable. +# CLI flag: -memberlist.message-history-buffer-bytes +[message_history_buffer_bytes: | default = 0] + # IP address to listen on for gossip messages. Multiple addresses may be # specified. Defaults to 0.0.0.0 # CLI flag: -memberlist.bind-addr diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index d66b581e19..1b2f512c35 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -465,6 +465,6 @@ func (t *Cortex) readyHandler(sm *services.Manager) http.HandlerFunc { } } - http.Error(w, "ready", http.StatusOK) + util.WriteTextResponse(w, "ready") } } diff --git a/pkg/ring/kv/memberlist/kv_init_service.go b/pkg/ring/kv/memberlist/kv_init_service.go index cdb20fd0a6..1400f23322 100644 --- a/pkg/ring/kv/memberlist/kv_init_service.go +++ b/pkg/ring/kv/memberlist/kv_init_service.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/memberlist" "go.uber.org/atomic" + "github.com/cortexproject/cortex/pkg/ring/kv/codec" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/services" ) @@ -86,64 +87,103 @@ func (kvs *KVInitService) stopping(_ error) error { func (kvs *KVInitService) ServeHTTP(w http.ResponseWriter, req *http.Request) { kv := kvs.getKV() - var ml *memberlist.Memberlist - var store map[string]valueDesc - - if kv != nil { - ml = kv.memberlist - store = kv.storeCopy() + if kv == nil { + util.WriteTextResponse(w, "This Cortex instance doesn't use memberlist.") + return } const ( - downloadParam = "download" - viewParam = "view" - viewFormat = "format" + downloadKeyParam = "downloadKey" + viewKeyParam = "viewKey" + viewMsgParam = "viewMsg" ) if err := req.ParseForm(); err == nil { - if req.Form[downloadParam] != nil { - download(w, store, req.Form[downloadParam][0]) // Use first value, ignore the rest. + if req.Form[downloadKeyParam] != nil { + downloadKey(w, kv.storeCopy(), req.Form[downloadKeyParam][0]) // Use first value, ignore the rest. + return + } + + if req.Form[viewKeyParam] != nil { + viewKey(w, kv, kv.storeCopy(), req.Form[viewKeyParam][0], getFormat(req)) return } - if req.Form[viewParam] != nil { - format := "" - if len(req.Form[viewFormat]) > 0 { - format = req.Form[viewFormat][0] + if req.Form[viewMsgParam] != nil { + msgID, err := strconv.Atoi(req.Form[viewMsgParam][0]) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return } - view(w, kv, store, req.Form[viewParam][0], format) + sent, received := kv.getSentAndReceivedMessages() + + for _, m := range append(sent, received...) { + if m.ID == msgID { + viewMessage(w, kv, m, getFormat(req)) + return + } + } + + http.Error(w, "message not found", http.StatusNotFound) return } } - members := ml.Members() + members := kv.memberlist.Members() sort.Slice(members, func(i, j int) bool { return members[i].Name < members[j].Name }) + sent, received := kv.getSentAndReceivedMessages() + util.RenderHTTPResponse(w, pageData{ - Now: time.Now(), - Initialized: kv != nil, - Memberlist: ml, - SortedMembers: members, - Store: store, + Now: time.Now(), + Memberlist: kv.memberlist, + SortedMembers: members, + Store: kv.storeCopy(), + SentMessages: sent, + ReceivedMessages: received, }, pageTemplate, req) } -func view(w http.ResponseWriter, kv *KV, store map[string]valueDesc, key string, format string) { - if kv == nil || store == nil || store[key].value == nil { +func getFormat(req *http.Request) string { + const viewFormat = "format" + + format := "" + if len(req.Form[viewFormat]) > 0 { + format = req.Form[viewFormat][0] + } + return format +} + +func viewMessage(w http.ResponseWriter, kv *KV, msg message, format string) { + c := kv.GetCodec(msg.Pair.Codec) + if c == nil { + http.Error(w, "codec not found", http.StatusNotFound) + return + } + + formatValue(w, c, msg.Pair.Value, format) +} + +func viewKey(w http.ResponseWriter, kv *KV, store map[string]valueDesc, key string, format string) { + if store[key].value == nil { http.Error(w, "value not found", http.StatusNotFound) return } - codec := kv.GetCodec(store[key].codecID) - if codec == nil { + c := kv.GetCodec(store[key].codecID) + if c == nil { http.Error(w, "codec not found", http.StatusNotFound) return } - val, err := codec.Decode(store[key].value) + formatValue(w, c, store[key].value, format) +} + +func formatValue(w http.ResponseWriter, codec codec.Codec, value []byte, format string) { + val, err := codec.Decode(value) if err != nil { http.Error(w, fmt.Sprintf("failed to decode: %v", err), http.StatusInternalServerError) return @@ -169,8 +209,8 @@ func view(w http.ResponseWriter, kv *KV, store map[string]valueDesc, key string, } } -func download(w http.ResponseWriter, store map[string]valueDesc, key string) { - if store == nil || store[key].value == nil { +func downloadKey(w http.ResponseWriter, store map[string]valueDesc, key string) { + if store[key].value == nil { http.Error(w, "value not found", http.StatusNotFound) return } @@ -188,11 +228,12 @@ func download(w http.ResponseWriter, store map[string]valueDesc, key string) { } type pageData struct { - Now time.Time - Initialized bool - Memberlist *memberlist.Memberlist - SortedMembers []*memberlist.Node - Store map[string]valueDesc + Now time.Time + Memberlist *memberlist.Memberlist + SortedMembers []*memberlist.Node + Store map[string]valueDesc + SentMessages []message + ReceivedMessages []message } var pageTemplate = template.Must(template.New("webpage").Parse(pageContent)) @@ -208,7 +249,6 @@ const pageContent = `

Cortex Memberlist Status

Current time: {{ .Now }}

- {{ if .Initialized }}