Skip to content

Commit

Permalink
Memberlist: keep a buffer with sent and received messages for trouble…
Browse files Browse the repository at this point in the history 
…shooting (cortexproject#3581)

Memberlist client can now keep buffer with sent and received messages, and display them in the admin UI.

Signed-off-by: Peter Štibraný <[email protected]>
  • Loading branch information
@pstibrany
pstibrany authored Dec 11, 2020
1 parent 410bbbc commit 638c78c
Show file tree
Hide file tree
Showing 9 changed files with 343 additions and 71 deletions.
3 changes: 2 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

## master / unreleased

* [ENHANCEMENT] Memberlist: add status page with available details about memberlist-based KV store and memberlist cluster. It's also possible to view KV values in Go struct or JSON format, or download for inspection. #3575
* [ENHANCEMENT] Memberlist: add status page (/memberlist) with available details about memberlist-based KV store and memberlist cluster. It's also possible to view KV values in Go struct or JSON format, or download for inspection. #3575
* [ENHANCEMENT] Memberlist: client can now keep a size-bounded buffer with sent and received messages and display them in the admin UI (/memberlist) for troubleshooting. #3581

## 1.6.0-rc.0 in progress

Expand Down
5 changes: 5 additions & 0 deletions docs/configuration/config-file-reference.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2979,6 +2979,11 @@ The `memberlist_config` configures the Gossip memberlist.
# CLI flag: -memberlist.leave-timeout
[leave_timeout: <duration> | default = 5s]
# How much space to use for keeping received and sent messages in memory for
# troubleshooting (two buffers). 0 to disable.
# CLI flag: -memberlist.message-history-buffer-bytes
[message_history_buffer_bytes: <int> | default = 0]
# IP address to listen on for gossip messages. Multiple addresses may be
# specified. Defaults to 0.0.0.0
# CLI flag: -memberlist.bind-addr
Expand Down
2 changes: 1 addition & 1 deletion pkg/cortex/cortex.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,6 @@ func (t *Cortex) readyHandler(sm *services.Manager) http.HandlerFunc {
}
}

http.Error(w, "ready", http.StatusOK)
util.WriteTextResponse(w, "ready")
}
}
184 changes: 142 additions & 42 deletions pkg/ring/kv/memberlist/kv_init_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/hashicorp/memberlist"
"go.uber.org/atomic"

"github.com/cortexproject/cortex/pkg/ring/kv/codec"
"github.com/cortexproject/cortex/pkg/util"
"github.com/cortexproject/cortex/pkg/util/services"
)
Expand Down Expand Up @@ -86,64 +87,103 @@ func (kvs *KVInitService) stopping(_ error) error {

func (kvs *KVInitService) ServeHTTP(w http.ResponseWriter, req *http.Request) {
kv := kvs.getKV()
var ml *memberlist.Memberlist
var store map[string]valueDesc

if kv != nil {
ml = kv.memberlist
store = kv.storeCopy()
if kv == nil {
util.WriteTextResponse(w, "This Cortex instance doesn't use memberlist.")
return
}

const (
downloadParam = "download"
viewParam = "view"
viewFormat = "format"
downloadKeyParam = "downloadKey"
viewKeyParam = "viewKey"
viewMsgParam = "viewMsg"
)

if err := req.ParseForm(); err == nil {
if req.Form[downloadParam] != nil {
download(w, store, req.Form[downloadParam][0]) // Use first value, ignore the rest.
if req.Form[downloadKeyParam] != nil {
downloadKey(w, kv.storeCopy(), req.Form[downloadKeyParam][0]) // Use first value, ignore the rest.
return
}

if req.Form[viewKeyParam] != nil {
viewKey(w, kv, kv.storeCopy(), req.Form[viewKeyParam][0], getFormat(req))
return
}

if req.Form[viewParam] != nil {
format := ""
if len(req.Form[viewFormat]) > 0 {
format = req.Form[viewFormat][0]
if req.Form[viewMsgParam] != nil {
msgID, err := strconv.Atoi(req.Form[viewMsgParam][0])
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}

view(w, kv, store, req.Form[viewParam][0], format)
sent, received := kv.getSentAndReceivedMessages()

for _, m := range append(sent, received...) {
if m.ID == msgID {
viewMessage(w, kv, m, getFormat(req))
return
}
}

http.Error(w, "message not found", http.StatusNotFound)
return
}
}

members := ml.Members()
members := kv.memberlist.Members()
sort.Slice(members, func(i, j int) bool {
return members[i].Name < members[j].Name
})

sent, received := kv.getSentAndReceivedMessages()

util.RenderHTTPResponse(w, pageData{
Now: time.Now(),
Initialized: kv != nil,
Memberlist: ml,
SortedMembers: members,
Store: store,
Now: time.Now(),
Memberlist: kv.memberlist,
SortedMembers: members,
Store: kv.storeCopy(),
SentMessages: sent,
ReceivedMessages: received,
}, pageTemplate, req)
}

func view(w http.ResponseWriter, kv *KV, store map[string]valueDesc, key string, format string) {
if kv == nil || store == nil || store[key].value == nil {
func getFormat(req *http.Request) string {
const viewFormat = "format"

format := ""
if len(req.Form[viewFormat]) > 0 {
format = req.Form[viewFormat][0]
}
return format
}

func viewMessage(w http.ResponseWriter, kv *KV, msg message, format string) {
c := kv.GetCodec(msg.Pair.Codec)
if c == nil {
http.Error(w, "codec not found", http.StatusNotFound)
return
}

formatValue(w, c, msg.Pair.Value, format)
}

func viewKey(w http.ResponseWriter, kv *KV, store map[string]valueDesc, key string, format string) {
if store[key].value == nil {
http.Error(w, "value not found", http.StatusNotFound)
return
}

codec := kv.GetCodec(store[key].codecID)
if codec == nil {
c := kv.GetCodec(store[key].codecID)
if c == nil {
http.Error(w, "codec not found", http.StatusNotFound)
return
}

val, err := codec.Decode(store[key].value)
formatValue(w, c, store[key].value, format)
}

func formatValue(w http.ResponseWriter, codec codec.Codec, value []byte, format string) {
val, err := codec.Decode(value)
if err != nil {
http.Error(w, fmt.Sprintf("failed to decode: %v", err), http.StatusInternalServerError)
return
Expand All @@ -169,8 +209,8 @@ func view(w http.ResponseWriter, kv *KV, store map[string]valueDesc, key string,
}
}

func download(w http.ResponseWriter, store map[string]valueDesc, key string) {
if store == nil || store[key].value == nil {
func downloadKey(w http.ResponseWriter, store map[string]valueDesc, key string) {
if store[key].value == nil {
http.Error(w, "value not found", http.StatusNotFound)
return
}
Expand All @@ -188,11 +228,12 @@ func download(w http.ResponseWriter, store map[string]valueDesc, key string) {
}

type pageData struct {
Now time.Time
Initialized bool
Memberlist *memberlist.Memberlist
SortedMembers []*memberlist.Node
Store map[string]valueDesc
Now time.Time
Memberlist *memberlist.Memberlist
SortedMembers []*memberlist.Node
Store map[string]valueDesc
SentMessages []message
ReceivedMessages []message
}

var pageTemplate = template.Must(template.New("webpage").Parse(pageContent))
Expand All @@ -208,7 +249,6 @@ const pageContent = `
<h1>Cortex Memberlist Status</h1>
<p>Current time: {{ .Now }}</p>
{{ if .Initialized }}
<ul>
<li>Health Score: {{ .Memberlist.GetHealthScore }} (lower = better, 0 = healthy)</li>
<li>Members: {{ .Memberlist.NumMembers }}</li>
Expand All @@ -231,10 +271,10 @@ const pageContent = `
<td>{{ $k }}</td>
<td>{{ $v }}</td>
<td>
<a href="?view={{ $k }}&format=json">json</a>
| <a href="?view={{ $k }}&format=json-pretty">json-pretty</a>
| <a href="?view={{ $k }}&format=struct">struct</a>
| <a href="?download={{ $k }}">download</a>
<a href="?viewKey={{ $k }}&format=json">json</a>
| <a href="?viewKey={{ $k }}&format=json-pretty">json-pretty</a>
| <a href="?viewKey={{ $k }}&format=struct">struct</a>
| <a href="?downloadKey={{ $k }}">download</a>
</td>
</tr>
{{ end }}
Expand Down Expand Up @@ -267,8 +307,68 @@ const pageContent = `
<p>State: 0 = Alive, 1 = Suspect, 2 = Dead, 3 = Left</p>
{{ else }}
<p>This Cortex instance doesn't use memberlist.</p>
{{ end }}
<h2>Received Messages</h2>
<table width="100%" border="1">
<thead>
<tr>
<th>ID</th>
<th>Time</th>
<th>Key</th>
<th>Value in the Message</th>
<th>Version After Update (0 = no change)</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
{{ range .ReceivedMessages }}
<tr>
<td>{{ .ID }}</td>
<td>{{ .Time.Format "15:04:05.000" }}</td>
<td>{{ .Pair.Key }}</td>
<td>size: {{ .Pair.Value | len }}, codec: {{ .Pair.Codec }}</td>
<td>{{ .Version }}</td>
<td>
<a href="?viewMsg={{ .ID }}&format=json">json</a>
| <a href="?viewMsg={{ .ID }}&format=json-pretty">json-pretty</a>
| <a href="?viewMsg={{ .ID }}&format=struct">struct</a>
</td>
</tr>
{{ end }}
</tbody>
</table>
<h2>Sent Messages</h2>
<table width="100%" border="1">
<thead>
<tr>
<th>ID</th>
<th>Time</th>
<th>Key</th>
<th>Value</th>
<th>Version</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
{{ range .SentMessages }}
<tr>
<td>{{ .ID }}</td>
<td>{{ .Time.Format "15:04:05.000" }}</td>
<td>{{ .Pair.Key }}</td>
<td>size: {{ .Pair.Value | len }}, codec: {{ .Pair.Codec }}</td>
<td>{{ .Version }}</td>
<td>
<a href="?viewMsg={{ .ID }}&format=json">json</a>
| <a href="?viewMsg={{ .ID }}&format=json-pretty">json-pretty</a>
| <a href="?viewMsg={{ .ID }}&format=struct">struct</a>
</td>
</tr>
{{ end }}
</tbody>
</table>
</body>
</html>`
32 changes: 23 additions & 9 deletions pkg/ring/kv/memberlist/kv_init_service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,6 @@ import (
)

func TestPage(t *testing.T) {
require.NoError(t, pageTemplate.Execute(&bytes.Buffer{}, pageData{
Now: time.Now(),
Initialized: false,
Memberlist: nil,
SortedMembers: nil,
Store: nil,
}))

conf := memberlist.DefaultLANConfig()
ml, err := memberlist.Create(conf)
require.NoError(t, err)
Expand All @@ -28,9 +20,31 @@ func TestPage(t *testing.T) {

require.NoError(t, pageTemplate.Execute(&bytes.Buffer{}, pageData{
Now: time.Now(),
Initialized: true,
Memberlist: ml,
SortedMembers: ml.Members(),
Store: nil,
ReceivedMessages: []message{{
ID: 10,
Time: time.Now(),
Size: 50,
Pair: KeyValuePair{
Key: "hello",
Value: []byte("world"),
Codec: "coded",
},
Version: 20,
}},

SentMessages: []message{{
ID: 10,
Time: time.Now(),
Size: 50,
Pair: KeyValuePair{
Key: "hello",
Value: []byte("world"),
Codec: "coded",
},
Version: 20,
}},
}))
}
Loading

0 comments on commit 638c78c

Please sign in to comment.