Skip to content

Commit

Permalink
Query Frontend: Add tenant label to metrics (#6887)
Browse files Browse the repository at this point in the history
This commit adds a tenant label to the HTTP metrics which are exported
by the Query Frontend.

Signed-off-by: Jacob Baungard Hansen <[email protected]>
  • Loading branch information
jacobbaungard authored Jan 2, 2024
1 parent 6d7abb5 commit 0fc308c
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 1 deletion.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
- [#6972](https://github.com/thanos-io/thanos/pull/6972) Store Gateway: Apply series limit when streaming series for series actually matched if lazy postings is enabled.
- [#6984](https://github.com/thanos-io/thanos/pull/6984) Store Gateway: Added `--store.index-header-lazy-download-strategy` to specify how to lazily download index headers when lazy mmap is enabled.

- [#6887](https://github.com/thanos-io/thanos/pull/6887) Query Frontend: *breaking :warning:* Add tenant label to relevant exported metrics. Note that this change may cause some pre-existing custom dashboard queries to be incorrect due to the added label.

### Changed

### Removed
Expand Down
2 changes: 1 addition & 1 deletion cmd/thanos/query_frontend.go
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ func runQueryFrontend(

// Configure Request Logging for HTTP calls.
logMiddleware := logging.NewHTTPServerMiddleware(logger, httpLogOpts...)
ins := extpromhttp.NewInstrumentationMiddleware(reg, nil)
ins := extpromhttp.NewTenantInstrumentationMiddleware(cfg.TenantHeader, cfg.DefaultTenant, reg, nil)

// Start metrics HTTP server.
{
Expand Down
109 changes: 109 additions & 0 deletions test/e2e/query_frontend_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -976,3 +976,112 @@ func (u tenantRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
r.Header.Set(u.tenantHeader, u.tenant)
return u.rt.RoundTrip(r)
}

func TestTenantQFEHTTPMetrics(t *testing.T) {
t.Parallel()

e, err := e2e.NewDockerEnvironment("tenant-metrics")
testutil.Ok(t, err)
t.Cleanup(e2ethanos.CleanScenario(t, e))

// scrape the local prometheus, and our querier metrics
prom1, sidecar1 := e2ethanos.NewPrometheusWithSidecar(e, "alone", e2ethanos.DefaultPromConfig("prom-alone", 0, "", "", e2ethanos.LocalPrometheusTarget, "tenant-metrics-querier-1:8080"), "", e2ethanos.DefaultPrometheusImage(), "")

q := e2ethanos.NewQuerierBuilder(e, "1", sidecar1.InternalEndpoint("grpc")).Init()
testutil.Ok(t, e2e.StartAndWaitReady(q))

inMemoryCacheConfig := queryfrontend.CacheProviderConfig{
Type: queryfrontend.INMEMORY,
Config: queryfrontend.InMemoryResponseCacheConfig{
MaxSizeItems: 1000,
Validity: time.Hour,
},
}

cfg := queryfrontend.Config{}
queryFrontend := e2ethanos.NewQueryFrontend(e, "1", "http://"+q.InternalEndpoint("http"), cfg, inMemoryCacheConfig)

ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
t.Cleanup(cancel)

testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, queryFrontend))

// Query once with default-tenant to ensure everything is ready
// for the following requests
instantQuery(t, ctx, queryFrontend.Endpoint("http"), func() string {
return "prometheus_api_remote_read_queries"
}, time.Now, promclient.QueryOptions{
Deduplicate: true,
}, 1)
testutil.Ok(t, err)

// Query a few times with tenant 1
instantQuery(t, ctx, queryFrontend.Endpoint("http"), func() string {
return "prometheus_api_remote_read_queries"
}, time.Now, promclient.QueryOptions{
Deduplicate: true,
HTTPHeaders: map[string][]string{"thanos-tenant": {"test-tenant-1"}},
}, 1)
testutil.Ok(t, err)

instantQuery(t, ctx, queryFrontend.Endpoint("http"), func() string {
return "go_goroutines"
}, time.Now, promclient.QueryOptions{
Deduplicate: true,
HTTPHeaders: map[string][]string{"thanos-tenant": {"test-tenant-1"}},
}, 2)
testutil.Ok(t, err)

instantQuery(t, ctx, queryFrontend.Endpoint("http"), func() string {
return "go_memstats_frees_total"
}, time.Now, promclient.QueryOptions{
Deduplicate: true,
HTTPHeaders: map[string][]string{"thanos-tenant": {"test-tenant-1"}},
}, 2)
testutil.Ok(t, err)

// query just once with tenant-2
instantQuery(t, ctx, queryFrontend.Endpoint("http"), func() string {
return "go_memstats_heap_alloc_bytes"
}, time.Now, promclient.QueryOptions{
Deduplicate: true,
HTTPHeaders: map[string][]string{"thanos-tenant": {"test-tenant-2"}},
}, 2)
testutil.Ok(t, err)

// check that http metrics for tenant-1 matches 3 requests, both for querier and query frontend
tenant1Matcher, err := matchers.NewMatcher(matchers.MatchEqual, "tenant", "test-tenant-1")
testutil.Ok(t, err)
testutil.Ok(t, q.WaitSumMetricsWithOptions(
e2emon.GreaterOrEqual(3),
[]string{"http_requests_total"}, e2emon.WithLabelMatchers(
tenant1Matcher,
),
e2emon.WaitMissingMetrics(),
))
testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions(
e2emon.GreaterOrEqual(3),
[]string{"http_requests_total"}, e2emon.WithLabelMatchers(
tenant1Matcher,
),
e2emon.WaitMissingMetrics(),
))

// check that http metrics for tenant-2 matches 1 requests, both for querier and query frontend
tenant2Matcher, err := matchers.NewMatcher(matchers.MatchEqual, "tenant", "test-tenant-2")
testutil.Ok(t, err)
testutil.Ok(t, q.WaitSumMetricsWithOptions(
e2emon.Equals(1),
[]string{"http_requests_total"}, e2emon.WithLabelMatchers(
tenant2Matcher,
),
e2emon.WaitMissingMetrics(),
))
testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions(
e2emon.Equals(1),
[]string{"http_requests_total"}, e2emon.WithLabelMatchers(
tenant2Matcher,
),
e2emon.WaitMissingMetrics(),
))
}

0 comments on commit 0fc308c

Please sign in to comment.