From 59b4e992e8d1bf647dee4301dd2aa4a1df505eb6 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Mon, 4 Nov 2024 14:08:55 +0900 Subject: [PATCH] Add v1 test porting to v2 Signed-off-by: SungJin1212 --- integration/e2e/util.go | 38 + integration/query_fuzz_test.go | 160 ++ pkg/cortexpbv2/compatv2.go | 17 +- pkg/distributor/distributor.go | 17 +- pkg/distributor/distributor_prw2_test.go | 1910 ++++++++++++++++++++++ pkg/distributor/distributor_test.go | 877 +++------- pkg/distributor/stats.go | 20 +- pkg/distributor/stats_test.go | 41 + pkg/ingester/ingester.go | 13 +- pkg/ingester/ingester_prw2_test.go | 1462 ++++++++++++++++- 10 files changed, 3845 insertions(+), 710 deletions(-) create mode 100644 pkg/distributor/distributor_prw2_test.go create mode 100644 pkg/distributor/stats_test.go diff --git a/integration/e2e/util.go b/integration/e2e/util.go index f257010613e..03038ad5ce1 100644 --- a/integration/e2e/util.go +++ b/integration/e2e/util.go @@ -311,6 +311,44 @@ func GenerateSeriesV2WithSamples( } } +func GenerateSeriesWithSamplesV2( + name string, + startTime time.Time, + scrapeInterval time.Duration, + startValue int, + numSamples int, + additionalLabels ...prompb.Label, +) (symbols []string, series cortexpbv2.TimeSeries) { + tsMillis := TimeToMilliseconds(startTime) + durMillis := scrapeInterval.Milliseconds() + + st := writev2.NewSymbolTable() + st.Symbolize("__name__") + st.Symbolize(name) + + lbls := labels.Labels{{Name: labels.MetricName, Value: name}} + for _, label := range additionalLabels { + st.Symbolize(label.Name) + st.Symbolize(label.Value) + lbls = append(lbls, labels.Label{Name: label.Name, Value: label.Value}) + } + + startTMillis := tsMillis + samples := make([]cortexpbv2.Sample, numSamples) + for i := 0; i < numSamples; i++ { + samples[i] = cortexpbv2.Sample{ + Timestamp: startTMillis, + Value: float64(i + startValue), + } + startTMillis += durMillis + } + + return st.Symbols(), cortexpbv2.TimeSeries{ + LabelsRefs: cortexpbv2.GetLabelsRefsFromLabels(st.Symbols(), lbls), + Samples: samples, + } +} + func GenerateSeriesWithSamples( name string, startTime time.Time, diff --git a/integration/query_fuzz_test.go b/integration/query_fuzz_test.go index a9d15a7ea96..9209ef9e613 100644 --- a/integration/query_fuzz_test.go +++ b/integration/query_fuzz_test.go @@ -16,6 +16,7 @@ import ( "testing" "time" + "github.com/cortexproject/cortex/pkg/cortexpbv2" "github.com/cortexproject/promqlsmith" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -52,6 +53,165 @@ func init() { } } +func TestRemoteWriteV1AndV2QueryResultFuzz(t *testing.T) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + // Start dependencies. + consul1 := e2edb.NewConsulWithName("consul1") + consul2 := e2edb.NewConsulWithName("consul2") + require.NoError(t, s.StartAndWaitReady(consul1, consul2)) + + flags := mergeFlags( + AlertmanagerLocalFlags(), + map[string]string{ + "-store.engine": blocksStorageEngine, + "-blocks-storage.backend": "filesystem", + "-blocks-storage.tsdb.head-compaction-interval": "4m", + "-blocks-storage.tsdb.block-ranges-period": "2h", + "-blocks-storage.tsdb.ship-interval": "1h", + "-blocks-storage.bucket-store.sync-interval": "15m", + "-blocks-storage.tsdb.retention-period": "2h", + "-blocks-storage.bucket-store.index-cache.backend": tsdb.IndexCacheBackendInMemory, + "-querier.query-store-for-labels-enabled": "true", + // Ingester. + "-ring.store": "consul", + // Distributor. + "-distributor.replication-factor": "1", + // Store-gateway. + "-store-gateway.sharding-enabled": "false", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + }, + ) + + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + + path1 := path.Join(s.SharedDir(), "cortex-1") + path2 := path.Join(s.SharedDir(), "cortex-2") + + flags1 := mergeFlags(flags, map[string]string{ + "-blocks-storage.filesystem.dir": path1, + "-consul.hostname": consul1.NetworkHTTPEndpoint(), + }) + flags2 := mergeFlags(flags, map[string]string{ + "-blocks-storage.filesystem.dir": path2, + "-consul.hostname": consul2.NetworkHTTPEndpoint(), + }) + // Start Cortex replicas. + cortex1 := e2ecortex.NewSingleBinary("cortex-1", flags1, "") + cortex2 := e2ecortex.NewSingleBinary("cortex-2", flags2, "") + require.NoError(t, s.StartAndWaitReady(cortex1, cortex2)) + + // Wait until Cortex replicas have updated the ring state. + require.NoError(t, cortex1.WaitSumMetrics(e2e.Equals(float64(512)), "cortex_ring_tokens_total")) + require.NoError(t, cortex2.WaitSumMetrics(e2e.Equals(float64(512)), "cortex_ring_tokens_total")) + + c1, err := e2ecortex.NewClient(cortex1.HTTPEndpoint(), cortex1.HTTPEndpoint(), "", "", "user-1") + require.NoError(t, err) + c2, err := e2ecortex.NewClient(cortex2.HTTPEndpoint(), cortex2.HTTPEndpoint(), "", "", "user-1") + require.NoError(t, err) + + now := time.Now() + // Push some series to Cortex. + start := now.Add(-time.Minute * 60) + scrapeInterval := 30 * time.Second + + numSeries := 10 + numSamples := 120 + serieses := make([]prompb.TimeSeries, numSeries) + seriesesV2 := make([]cortexpbv2.TimeSeries, numSeries) + lbls := make([]labels.Labels, numSeries) + + // make v1 series + for i := 0; i < numSeries; i++ { + series := e2e.GenerateSeriesWithSamples("test_series", start, scrapeInterval, i*numSamples, numSamples, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "series", Value: strconv.Itoa(i)}) + serieses[i] = series + + builder := labels.NewBuilder(labels.EmptyLabels()) + for _, lbl := range series.Labels { + builder.Set(lbl.Name, lbl.Value) + } + lbls[i] = builder.Labels() + } + // make v2 series + for i := 0; i < numSeries; i++ { + series := e2e.GenerateSeriesWithSamplesV2("test_series", start, scrapeInterval, i*numSamples, numSamples, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "series", Value: strconv.Itoa(i)}) + seriesesV2[i] = series + } + + res, err := c1.Push(serieses) + require.NoError(t, err) + require.Equal(t, 200, res.StatusCode) + + res, err = c2.PushV2(seriesesV2) + require.NoError(t, err) + require.Equal(t, 200, res.StatusCode) + + waitUntilReady(t, context.Background(), c1, c2, `{job="test"}`, start, now) + + rnd := rand.New(rand.NewSource(now.Unix())) + opts := []promqlsmith.Option{ + promqlsmith.WithEnableOffset(true), + promqlsmith.WithEnableAtModifier(true), + promqlsmith.WithEnabledFunctions(enabledFunctions), + } + ps := promqlsmith.New(rnd, lbls, opts...) + + type testCase struct { + query string + res1, res2 model.Value + err1, err2 error + } + + queryStart := now.Add(-time.Minute * 50) + queryEnd := now.Add(-time.Minute * 10) + cases := make([]*testCase, 0, 500) + testRun := 500 + var ( + expr parser.Expr + query string + ) + for i := 0; i < testRun; i++ { + for { + expr = ps.WalkRangeQuery() + query = expr.Pretty(0) + // timestamp is a known function that break with disable chunk trimming. + if isValidQuery(expr, 5) && !strings.Contains(query, "timestamp") { + break + } + } + res1, err1 := c1.QueryRange(query, queryStart, queryEnd, scrapeInterval) + res2, err2 := c2.QueryRange(query, queryStart, queryEnd, scrapeInterval) + cases = append(cases, &testCase{ + query: query, + res1: res1, + res2: res2, + err1: err1, + err2: err2, + }) + } + + failures := 0 + for i, tc := range cases { + qt := "range query" + if tc.err1 != nil || tc.err2 != nil { + if !cmp.Equal(tc.err1, tc.err2) { + t.Logf("case %d error mismatch.\n%s: %s\nerr1: %v\nerr2: %v\n", i, qt, tc.query, tc.err1, tc.err2) + failures++ + } + } else if !cmp.Equal(tc.res1, tc.res2, comparer) { + t.Logf("case %d results mismatch.\n%s: %s\nres1: %s\nres2: %s\n", i, qt, tc.query, tc.res1.String(), tc.res2.String()) + failures++ + } + } + if failures > 0 { + require.Failf(t, "finished query fuzzing tests", "%d test cases failed", failures) + } +} + func TestDisableChunkTrimmingFuzz(t *testing.T) { s, err := e2e.NewScenario(networkName) require.NoError(t, err) diff --git a/pkg/cortexpbv2/compatv2.go b/pkg/cortexpbv2/compatv2.go index 781b7fbe4b2..7ecab6702b9 100644 --- a/pkg/cortexpbv2/compatv2.go +++ b/pkg/cortexpbv2/compatv2.go @@ -2,12 +2,27 @@ package cortexpbv2 import ( "github.com/prometheus/prometheus/model/labels" + writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2" "github.com/cortexproject/cortex/pkg/cortexpb" ) // ToWriteRequestV2 converts matched slices of Labels, Samples, and Histograms into a WriteRequest proto. -func ToWriteRequestV2(lbls []labels.Labels, symbols []string, samples []Sample, histograms []Histogram, metadata []Metadata, source WriteRequest_SourceEnum) *WriteRequest { +func ToWriteRequestV2(lbls []labels.Labels, samples []Sample, histograms []Histogram, metadata []Metadata, source WriteRequest_SourceEnum, additionalSymbols ...string) *WriteRequest { + st := writev2.NewSymbolTable() + for _, lbl := range lbls { + lbl.Range(func(l labels.Label) { + st.Symbolize(l.Name) + st.Symbolize(l.Value) + }) + } + + for _, s := range additionalSymbols { + st.Symbolize(s) + } + + symbols := st.Symbols() + req := &WriteRequest{ Symbols: symbols, Source: source, diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index da5fe7897b3..436c42b9750 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -768,9 +768,18 @@ func (d *Distributor) sendV2(ctx context.Context, symbols []string, ingester rin d.ingesterAppendFailures.WithLabelValues(id, typeSamples, getErrorStatus(err)).Inc() } - d.ingesterAppends.WithLabelValues(id, typeMetadata).Inc() - if err != nil { - d.ingesterAppendFailures.WithLabelValues(id, typeMetadata, getErrorStatus(err)).Inc() + metadataAppend := false + for _, ts := range timeseries { + if ts.Metadata.Type != cortexpbv2.METRIC_TYPE_UNSPECIFIED { + metadataAppend = true + break + } + } + if metadataAppend { + d.ingesterAppends.WithLabelValues(id, typeMetadata).Inc() + if err != nil { + d.ingesterAppendFailures.WithLabelValues(id, typeMetadata, getErrorStatus(err)).Inc() + } } } @@ -926,7 +935,7 @@ func (d *Distributor) PushV2(ctx context.Context, req *cortexpbv2.WriteRequest) // Return a 429 here to tell the client it is going too fast. // Client may discard the data or slow down and re-send. // Prometheus v2.26 added a remote-write option 'retry_on_http_429'. - return nil, httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (%v) exceeded while adding %d samples", d.ingestionRateLimiter.Limit(now, userID), totalSamples) + return nil, httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (%v) exceeded while adding %d samples and %d metadata", d.ingestionRateLimiter.Limit(now, userID), totalSamples, validatedMetadatas) } // totalN included samples and metadata. Ingester follows this pattern when computing its ingestion rate. diff --git a/pkg/distributor/distributor_prw2_test.go b/pkg/distributor/distributor_prw2_test.go new file mode 100644 index 00000000000..65ef0328a7d --- /dev/null +++ b/pkg/distributor/distributor_prw2_test.go @@ -0,0 +1,1910 @@ +package distributor + +import ( + "context" + "fmt" + "math" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" + writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2" + "github.com/prometheus/prometheus/tsdb/tsdbutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/weaveworks/common/httpgrpc" + "github.com/weaveworks/common/user" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/cortexproject/cortex/pkg/cortexpb" + "github.com/cortexproject/cortex/pkg/cortexpbv2" + "github.com/cortexproject/cortex/pkg/ingester" + "github.com/cortexproject/cortex/pkg/ring" + "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/chunkcompat" + "github.com/cortexproject/cortex/pkg/util/flagext" + "github.com/cortexproject/cortex/pkg/util/limiter" + "github.com/cortexproject/cortex/pkg/util/test" + "github.com/cortexproject/cortex/pkg/util/validation" +) + +var ( + emptyResponseV2 = &cortexpbv2.WriteResponse{} +) + +// TODO(Sungjin1212): Add PushHAInstances Test after implement PRW2 HA tracker +// TODO(Sungjin1212): Add TestDistributor_Push_LabelRemoval, TestDistributor_Push_LabelRemoval_RemovingNameLabelWillError Test after implement PRW2 relabel + +func TestDistributorPRW2_Push(t *testing.T) { + t.Parallel() + // Metrics to assert on. + lastSeenTimestamp := "cortex_distributor_latest_seen_sample_timestamp_seconds" + distributorAppend := "cortex_distributor_ingester_appends_total" + distributorAppendFailure := "cortex_distributor_ingester_append_failures_total" + distributorReceivedSamples := "cortex_distributor_received_samples_total" + ctx := user.InjectOrgID(context.Background(), "userDistributorPush") + + type samplesIn struct { + num int + startTimestampMs int64 + } + for name, tc := range map[string]struct { + metricNames []string + numIngesters int + happyIngesters int + samples samplesIn + histogramSamples bool + metadata int + expectedResponse *cortexpbv2.WriteResponse + expectedError error + expectedMetrics string + ingesterError error + }{ + "A push of no samples shouldn't block or return error, even if ingesters are sad": { + numIngesters: 3, + happyIngesters: 0, + expectedResponse: emptyResponseV2, + }, + "A push to 3 happy ingesters should succeed": { + numIngesters: 3, + happyIngesters: 3, + samples: samplesIn{num: 5, startTimestampMs: 123456789000}, + metadata: 5, + expectedResponse: emptyResponseV2, + metricNames: []string{lastSeenTimestamp}, + expectedMetrics: ` + # HELP cortex_distributor_latest_seen_sample_timestamp_seconds Unix timestamp of latest received sample per user. + # TYPE cortex_distributor_latest_seen_sample_timestamp_seconds gauge + cortex_distributor_latest_seen_sample_timestamp_seconds{user="userDistributorPush"} 123456789.004 + `, + }, + "A push to 2 happy ingesters should succeed": { + numIngesters: 3, + happyIngesters: 2, + samples: samplesIn{num: 5, startTimestampMs: 123456789000}, + metadata: 5, + expectedResponse: emptyResponseV2, + metricNames: []string{lastSeenTimestamp}, + expectedMetrics: ` + # HELP cortex_distributor_latest_seen_sample_timestamp_seconds Unix timestamp of latest received sample per user. + # TYPE cortex_distributor_latest_seen_sample_timestamp_seconds gauge + cortex_distributor_latest_seen_sample_timestamp_seconds{user="userDistributorPush"} 123456789.004 + `, + }, + "A push to 1 happy ingesters should fail": { + numIngesters: 3, + happyIngesters: 1, + samples: samplesIn{num: 10, startTimestampMs: 123456789000}, + expectedError: errFail, + metricNames: []string{lastSeenTimestamp}, + expectedMetrics: ` + # HELP cortex_distributor_latest_seen_sample_timestamp_seconds Unix timestamp of latest received sample per user. + # TYPE cortex_distributor_latest_seen_sample_timestamp_seconds gauge + cortex_distributor_latest_seen_sample_timestamp_seconds{user="userDistributorPush"} 123456789.009 + `, + }, + "A push to 0 happy ingesters should fail": { + numIngesters: 3, + happyIngesters: 0, + samples: samplesIn{num: 10, startTimestampMs: 123456789000}, + expectedError: errFail, + metricNames: []string{lastSeenTimestamp}, + expectedMetrics: ` + # HELP cortex_distributor_latest_seen_sample_timestamp_seconds Unix timestamp of latest received sample per user. + # TYPE cortex_distributor_latest_seen_sample_timestamp_seconds gauge + cortex_distributor_latest_seen_sample_timestamp_seconds{user="userDistributorPush"} 123456789.009 + `, + }, + "A push exceeding burst size should fail": { + numIngesters: 3, + happyIngesters: 3, + samples: samplesIn{num: 25, startTimestampMs: 123456789000}, + metadata: 5, + expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (20) exceeded while adding 25 samples and 5 metadata"), + metricNames: []string{lastSeenTimestamp}, + expectedMetrics: ` + # HELP cortex_distributor_latest_seen_sample_timestamp_seconds Unix timestamp of latest received sample per user. + # TYPE cortex_distributor_latest_seen_sample_timestamp_seconds gauge + cortex_distributor_latest_seen_sample_timestamp_seconds{user="userDistributorPush"} 123456789.024 + `, + }, + "A push to ingesters should report the correct metrics with no metadata": { + numIngesters: 3, + happyIngesters: 2, + samples: samplesIn{num: 1, startTimestampMs: 123456789000}, + metadata: 0, + metricNames: []string{distributorAppend, distributorAppendFailure}, + expectedResponse: emptyResponseV2, + expectedMetrics: ` + # HELP cortex_distributor_ingester_append_failures_total The total number of failed batch appends sent to ingesters. + # TYPE cortex_distributor_ingester_append_failures_total counter + cortex_distributor_ingester_append_failures_total{ingester="ingester-2",status="5xx",type="samples"} 1 + # HELP cortex_distributor_ingester_appends_total The total number of batch appends sent to ingesters. + # TYPE cortex_distributor_ingester_appends_total counter + cortex_distributor_ingester_appends_total{ingester="ingester-0",type="samples"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-1",type="samples"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-2",type="samples"} 1 + `, + }, + "A push to ingesters should report samples and metadata metrics with no samples": { + numIngesters: 3, + happyIngesters: 2, + samples: samplesIn{num: 0, startTimestampMs: 123456789000}, + metadata: 1, + metricNames: []string{distributorAppend, distributorAppendFailure}, + expectedResponse: emptyResponseV2, + ingesterError: httpgrpc.Errorf(http.StatusInternalServerError, "Fail"), + expectedMetrics: ` + # HELP cortex_distributor_ingester_append_failures_total The total number of failed batch appends sent to ingesters. + # TYPE cortex_distributor_ingester_append_failures_total counter + cortex_distributor_ingester_append_failures_total{ingester="ingester-2",status="5xx",type="metadata"} 1 + cortex_distributor_ingester_append_failures_total{ingester="ingester-2",status="5xx",type="samples"} 1 + # HELP cortex_distributor_ingester_appends_total The total number of batch appends sent to ingesters. + # TYPE cortex_distributor_ingester_appends_total counter + cortex_distributor_ingester_appends_total{ingester="ingester-0",type="metadata"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-1",type="metadata"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-2",type="metadata"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-0",type="samples"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-1",type="samples"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-2",type="samples"} 1 + `, + }, + "A push to overloaded ingesters should report the correct metrics": { + numIngesters: 3, + happyIngesters: 2, + samples: samplesIn{num: 0, startTimestampMs: 123456789000}, + metadata: 1, + metricNames: []string{distributorAppend, distributorAppendFailure}, + expectedResponse: emptyResponseV2, + ingesterError: httpgrpc.Errorf(http.StatusTooManyRequests, "Fail"), + expectedMetrics: ` + # HELP cortex_distributor_ingester_appends_total The total number of batch appends sent to ingesters. + # TYPE cortex_distributor_ingester_appends_total counter + cortex_distributor_ingester_appends_total{ingester="ingester-0",type="metadata"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-1",type="metadata"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-2",type="metadata"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-0",type="samples"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-1",type="samples"} 1 + cortex_distributor_ingester_appends_total{ingester="ingester-2",type="samples"} 1 + # HELP cortex_distributor_ingester_append_failures_total The total number of failed batch appends sent to ingesters. + # TYPE cortex_distributor_ingester_append_failures_total counter + cortex_distributor_ingester_append_failures_total{ingester="ingester-2",status="4xx",type="metadata"} 1 + cortex_distributor_ingester_append_failures_total{ingester="ingester-2",status="4xx",type="samples"} 1 + `, + }, + "A push to 3 happy ingesters should succeed, histograms": { + numIngesters: 3, + happyIngesters: 3, + samples: samplesIn{num: 5, startTimestampMs: 123456789000}, + histogramSamples: true, + metadata: 5, + expectedResponse: emptyResponseV2, + metricNames: []string{lastSeenTimestamp, distributorReceivedSamples}, + expectedMetrics: ` + # HELP cortex_distributor_latest_seen_sample_timestamp_seconds Unix timestamp of latest received sample per user. + # TYPE cortex_distributor_latest_seen_sample_timestamp_seconds gauge + cortex_distributor_latest_seen_sample_timestamp_seconds{user="userDistributorPush"} 123456789.004 + # HELP cortex_distributor_received_samples_total The total number of received samples, excluding rejected and deduped samples. + # TYPE cortex_distributor_received_samples_total counter + cortex_distributor_received_samples_total{type="float",user="userDistributorPush"} 0 + cortex_distributor_received_samples_total{type="histogram",user="userDistributorPush"} 5 + `, + }, + "A push to 2 happy ingesters should succeed, histograms": { + numIngesters: 3, + happyIngesters: 2, + samples: samplesIn{num: 5, startTimestampMs: 123456789000}, + histogramSamples: true, + metadata: 5, + expectedResponse: emptyResponseV2, + metricNames: []string{lastSeenTimestamp, distributorReceivedSamples}, + expectedMetrics: ` + # HELP cortex_distributor_latest_seen_sample_timestamp_seconds Unix timestamp of latest received sample per user. + # TYPE cortex_distributor_latest_seen_sample_timestamp_seconds gauge + cortex_distributor_latest_seen_sample_timestamp_seconds{user="userDistributorPush"} 123456789.004 + # HELP cortex_distributor_received_samples_total The total number of received samples, excluding rejected and deduped samples. + # TYPE cortex_distributor_received_samples_total counter + cortex_distributor_received_samples_total{type="float",user="userDistributorPush"} 0 + cortex_distributor_received_samples_total{type="histogram",user="userDistributorPush"} 5 + `, + }, + "A push to 1 happy ingesters should fail, histograms": { + numIngesters: 3, + happyIngesters: 1, + samples: samplesIn{num: 10, startTimestampMs: 123456789000}, + histogramSamples: true, + expectedError: errFail, + metricNames: []string{lastSeenTimestamp, distributorReceivedSamples}, + expectedMetrics: ` + # HELP cortex_distributor_latest_seen_sample_timestamp_seconds Unix timestamp of latest received sample per user. + # TYPE cortex_distributor_latest_seen_sample_timestamp_seconds gauge + cortex_distributor_latest_seen_sample_timestamp_seconds{user="userDistributorPush"} 123456789.009 + # HELP cortex_distributor_received_samples_total The total number of received samples, excluding rejected and deduped samples. + # TYPE cortex_distributor_received_samples_total counter + cortex_distributor_received_samples_total{type="float",user="userDistributorPush"} 0 + cortex_distributor_received_samples_total{type="histogram",user="userDistributorPush"} 10 + `, + }, + "A push exceeding burst size should fail, histograms": { + numIngesters: 3, + happyIngesters: 3, + samples: samplesIn{num: 25, startTimestampMs: 123456789000}, + histogramSamples: true, + metadata: 5, + expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (20) exceeded while adding 25 samples and 5 metadata"), + metricNames: []string{lastSeenTimestamp, distributorReceivedSamples}, + expectedMetrics: ` + # HELP cortex_distributor_latest_seen_sample_timestamp_seconds Unix timestamp of latest received sample per user. + # TYPE cortex_distributor_latest_seen_sample_timestamp_seconds gauge + cortex_distributor_latest_seen_sample_timestamp_seconds{user="userDistributorPush"} 123456789.024 + # HELP cortex_distributor_received_samples_total The total number of received samples, excluding rejected and deduped samples. + # TYPE cortex_distributor_received_samples_total counter + cortex_distributor_received_samples_total{type="float",user="userDistributorPush"} 0 + cortex_distributor_received_samples_total{type="histogram",user="userDistributorPush"} 25 + `, + }, + } { + for _, shardByAllLabels := range []bool{true, false} { + tc := tc + name := name + shardByAllLabels := shardByAllLabels + t.Run(fmt.Sprintf("[%s](shardByAllLabels=%v)", name, shardByAllLabels), func(t *testing.T) { + t.Parallel() + limits := &validation.Limits{} + flagext.DefaultValues(limits) + limits.IngestionRate = 20 + limits.IngestionBurstSize = 20 + + ds, _, regs, _ := prepare(t, prepConfig{ + numIngesters: tc.numIngesters, + happyIngesters: tc.happyIngesters, + numDistributors: 1, + shardByAllLabels: shardByAllLabels, + limits: limits, + errFail: tc.ingesterError, + }) + + var request *cortexpbv2.WriteRequest + if !tc.histogramSamples { + request = makeWriteRequestV2WithSamples(tc.samples.startTimestampMs, tc.samples.num, tc.metadata) + } else { + request = makeWriteRequestV2WithHistogram(tc.samples.startTimestampMs, tc.samples.num, tc.metadata) + } + + response, err := ds[0].PushV2(ctx, request) + assert.Equal(t, tc.expectedResponse, response) + assert.Equal(t, status.Code(tc.expectedError), status.Code(err)) + + // Check tracked Prometheus metrics. Since the Push() response is sent as soon as the quorum + // is reached, when we reach this point the 3rd ingester may not have received series/metadata + // yet. To avoid flaky test we retry metrics assertion until we hit the desired state (no error) + // within a reasonable timeout. + if tc.expectedMetrics != "" { + test.Poll(t, time.Second, nil, func() interface{} { + return testutil.GatherAndCompare(regs[0], strings.NewReader(tc.expectedMetrics), tc.metricNames...) + }) + } + }) + } + } +} + +func TestDistributorPRW2_PushIngestionRateLimiter(t *testing.T) { + t.Parallel() + type testPush struct { + samples int + metadata int + expectedError error + } + + ctx := user.InjectOrgID(context.Background(), "user") + tests := map[string]struct { + distributors int + ingestionRateStrategy string + ingestionRate float64 + ingestionBurstSize int + pushes []testPush + }{ + "local strategy: limit should be set to each distributor": { + distributors: 2, + ingestionRateStrategy: validation.LocalIngestionRateStrategy, + ingestionRate: 10, + ingestionBurstSize: 10, + pushes: []testPush{ + {samples: 4, expectedError: nil}, + {metadata: 1, expectedError: nil}, + {samples: 6, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 6 samples and 0 metadata")}, + {samples: 4, metadata: 1, expectedError: nil}, + {samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 1 samples and 0 metadata")}, + {metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (10) exceeded while adding 0 samples and 1 metadata")}, + }, + }, + "global strategy: limit should be evenly shared across distributors": { + distributors: 2, + ingestionRateStrategy: validation.GlobalIngestionRateStrategy, + ingestionRate: 10, + ingestionBurstSize: 5, + pushes: []testPush{ + {samples: 2, expectedError: nil}, + {samples: 1, expectedError: nil}, + {samples: 2, metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 2 samples and 1 metadata")}, + {samples: 2, expectedError: nil}, + {samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 1 samples and 0 metadata")}, + {metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 0 samples and 1 metadata")}, + }, + }, + "global strategy: burst should set to each distributor": { + distributors: 2, + ingestionRateStrategy: validation.GlobalIngestionRateStrategy, + ingestionRate: 10, + ingestionBurstSize: 20, + pushes: []testPush{ + {samples: 10, expectedError: nil}, + {samples: 5, expectedError: nil}, + {samples: 5, metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 5 samples and 1 metadata")}, + {samples: 5, expectedError: nil}, + {samples: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 1 samples and 0 metadata")}, + {metadata: 1, expectedError: httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (5) exceeded while adding 0 samples and 1 metadata")}, + }, + }, + } + + for testName, testData := range tests { + testData := testData + + for _, enableHistogram := range []bool{false, true} { + enableHistogram := enableHistogram + t.Run(fmt.Sprintf("%s, histogram=%s", testName, strconv.FormatBool(enableHistogram)), func(t *testing.T) { + t.Parallel() + limits := &validation.Limits{} + flagext.DefaultValues(limits) + limits.IngestionRateStrategy = testData.ingestionRateStrategy + limits.IngestionRate = testData.ingestionRate + limits.IngestionBurstSize = testData.ingestionBurstSize + + // Start all expected distributors + distributors, _, _, _ := prepare(t, prepConfig{ + numIngesters: 3, + happyIngesters: 3, + numDistributors: testData.distributors, + shardByAllLabels: true, + limits: limits, + }) + + // Push samples in multiple requests to the first distributor + for _, push := range testData.pushes { + var request *cortexpbv2.WriteRequest + if !enableHistogram { + request = makeWriteRequestV2WithSamples(0, push.samples, push.metadata) + } else { + request = makeWriteRequestV2WithHistogram(0, push.samples, push.metadata) + } + response, err := distributors[0].PushV2(ctx, request) + + if push.expectedError == nil { + assert.Equal(t, emptyResponseV2, response) + assert.Nil(t, err) + } else { + assert.Nil(t, response) + assert.Equal(t, push.expectedError, err) + } + } + }) + } + } +} + +func TestPushPRW2_QuorumError(t *testing.T) { + t.Parallel() + + var limits validation.Limits + flagext.DefaultValues(&limits) + + limits.IngestionRate = math.MaxFloat64 + + dists, ingesters, _, r := prepare(t, prepConfig{ + numDistributors: 1, + numIngesters: 3, + happyIngesters: 0, + shuffleShardSize: 3, + shardByAllLabels: true, + shuffleShardEnabled: true, + limits: &limits, + }) + + ctx := user.InjectOrgID(context.Background(), "user") + + d := dists[0] + + // we should run several write request to make sure we dont have any race condition on the batchTracker#record code + numberOfWrites := 10000 + + // Using 429 just to make sure we are not hitting the &limits + // Simulating 2 4xx and 1 5xx -> Should return 4xx + ingesters[0].failResp.Store(httpgrpc.Errorf(429, "Throttling")) + ingesters[1].failResp.Store(httpgrpc.Errorf(500, "InternalServerError")) + ingesters[2].failResp.Store(httpgrpc.Errorf(429, "Throttling")) + + for i := 0; i < numberOfWrites; i++ { + request := makeWriteRequestV2WithSamples(0, 30, 20) + _, err := d.PushV2(ctx, request) + status, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.Code(429), status.Code()) + } + + // Simulating 2 5xx and 1 4xx -> Should return 5xx + ingesters[0].failResp.Store(httpgrpc.Errorf(500, "InternalServerError")) + ingesters[1].failResp.Store(httpgrpc.Errorf(429, "Throttling")) + ingesters[2].failResp.Store(httpgrpc.Errorf(500, "InternalServerError")) + + for i := 0; i < numberOfWrites; i++ { + request := makeWriteRequest(0, 300, 200, 10) + _, err := d.Push(ctx, request) + status, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.Code(500), status.Code()) + } + + // Simulating 2 different errors and 1 success -> This case we may return any of the errors + ingesters[0].failResp.Store(httpgrpc.Errorf(500, "InternalServerError")) + ingesters[1].failResp.Store(httpgrpc.Errorf(429, "Throttling")) + ingesters[2].happy.Store(true) + + for i := 0; i < numberOfWrites; i++ { + request := makeWriteRequest(0, 30, 20, 10) + _, err := d.Push(ctx, request) + status, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.Code(429), status.Code()) + } + + // Simulating 1 error -> Should return 2xx + ingesters[0].failResp.Store(httpgrpc.Errorf(500, "InternalServerError")) + ingesters[1].happy.Store(true) + ingesters[2].happy.Store(true) + + for i := 0; i < 1; i++ { + request := makeWriteRequest(0, 30, 20, 10) + _, err := d.Push(ctx, request) + require.NoError(t, err) + } + + // Simulating an unhealthy ingester (ingester 2) + ingesters[0].failResp.Store(httpgrpc.Errorf(500, "InternalServerError")) + ingesters[1].happy.Store(true) + ingesters[2].happy.Store(true) + + err := r.KVClient.CAS(context.Background(), ingester.RingKey, func(in interface{}) (interface{}, bool, error) { + r := in.(*ring.Desc) + ingester2 := r.Ingesters["ingester-2"] + ingester2.State = ring.LEFT + ingester2.Timestamp = time.Now().Unix() + r.Ingesters["ingester-2"] = ingester2 + return in, true, nil + }) + + require.NoError(t, err) + + // Give time to the ring get updated with the KV value + test.Poll(t, 15*time.Second, true, func() interface{} { + replicationSet, _ := r.GetAllHealthy(ring.Read) + return len(replicationSet.Instances) == 2 + }) + + for i := 0; i < numberOfWrites; i++ { + request := makeWriteRequest(0, 30, 20, 10) + _, err := d.Push(ctx, request) + require.Error(t, err) + status, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.Code(500), status.Code()) + } +} + +func TestDistributorPRW2_PushInstanceLimits(t *testing.T) { + t.Parallel() + + type testPush struct { + samples int + metadata int + expectedError error + } + + ctx := user.InjectOrgID(context.Background(), "user") + tests := map[string]struct { + preInflight int + preRateSamples int // initial rate before first push + pushes []testPush // rate is recomputed after each push + + // limits + inflightLimit int + ingestionRateLimit float64 + + metricNames []string + expectedMetrics string + }{ + "no limits limit": { + preInflight: 100, + preRateSamples: 1000, + + pushes: []testPush{ + {samples: 100, expectedError: nil}, + }, + + metricNames: []string{instanceLimitsMetric}, + expectedMetrics: ` + # HELP cortex_distributor_instance_limits Instance limits used by this distributor. + # TYPE cortex_distributor_instance_limits gauge + cortex_distributor_instance_limits{limit="max_inflight_push_requests"} 0 + cortex_distributor_instance_limits{limit="max_ingestion_rate"} 0 + `, + }, + "below inflight limit": { + preInflight: 100, + inflightLimit: 101, + pushes: []testPush{ + {samples: 100, expectedError: nil}, + }, + + metricNames: []string{instanceLimitsMetric, "cortex_distributor_inflight_push_requests"}, + expectedMetrics: ` + # HELP cortex_distributor_inflight_push_requests Current number of inflight push requests in distributor. + # TYPE cortex_distributor_inflight_push_requests gauge + cortex_distributor_inflight_push_requests 100 + + # HELP cortex_distributor_instance_limits Instance limits used by this distributor. + # TYPE cortex_distributor_instance_limits gauge + cortex_distributor_instance_limits{limit="max_inflight_push_requests"} 101 + cortex_distributor_instance_limits{limit="max_ingestion_rate"} 0 + `, + }, + "hits inflight limit": { + preInflight: 101, + inflightLimit: 101, + pushes: []testPush{ + {samples: 100, expectedError: errTooManyInflightPushRequests}, + }, + }, + "below ingestion rate limit": { + preRateSamples: 500, + ingestionRateLimit: 1000, + + pushes: []testPush{ + {samples: 1000, expectedError: nil}, + }, + + metricNames: []string{instanceLimitsMetric, "cortex_distributor_ingestion_rate_samples_per_second"}, + expectedMetrics: ` + # HELP cortex_distributor_ingestion_rate_samples_per_second Current ingestion rate in samples/sec that distributor is using to limit access. + # TYPE cortex_distributor_ingestion_rate_samples_per_second gauge + cortex_distributor_ingestion_rate_samples_per_second 600 + + # HELP cortex_distributor_instance_limits Instance limits used by this distributor. + # TYPE cortex_distributor_instance_limits gauge + cortex_distributor_instance_limits{limit="max_inflight_push_requests"} 0 + cortex_distributor_instance_limits{limit="max_ingestion_rate"} 1000 + `, + }, + "hits rate limit on first request, but second request can proceed": { + preRateSamples: 1200, + ingestionRateLimit: 1000, + + pushes: []testPush{ + {samples: 100, expectedError: errMaxSamplesPushRateLimitReached}, + {samples: 100, expectedError: nil}, + }, + }, + // TODO(Sungjin1212): enable test after v2 prealloc timeseries implement + //"below rate limit on first request, but hits the rate limit afterwards": { + // preRateSamples: 500, + // ingestionRateLimit: 1000, + // + // pushes: []testPush{ + // {samples: 5000, expectedError: nil}, // after push, rate = 500 + 0.2*(5000-500) = 1400 + // {samples: 5000, expectedError: errMaxSamplesPushRateLimitReached}, // after push, rate = 1400 + 0.2*(0 - 1400) = 1120 + // {samples: 5000, expectedError: errMaxSamplesPushRateLimitReached}, // after push, rate = 1120 + 0.2*(0 - 1120) = 896 + // {samples: 5000, expectedError: nil}, // 896 is below 1000, so this push succeeds, new rate = 896 + 0.2*(5000-896) = 1716.8 + // }, + //}, + } + + for testName, testData := range tests { + testData := testData + + for _, enableHistogram := range []bool{true, false} { + enableHistogram := enableHistogram + t.Run(fmt.Sprintf("%s, histogram=%s", testName, strconv.FormatBool(enableHistogram)), func(t *testing.T) { + t.Parallel() + limits := &validation.Limits{} + flagext.DefaultValues(limits) + + // Start all expected distributors + distributors, _, regs, _ := prepare(t, prepConfig{ + numIngesters: 3, + happyIngesters: 3, + numDistributors: 1, + shardByAllLabels: true, + limits: limits, + maxInflightRequests: testData.inflightLimit, + maxIngestionRate: testData.ingestionRateLimit, + }) + + d := distributors[0] + d.inflightPushRequests.Add(int64(testData.preInflight)) + d.ingestionRate.Add(int64(testData.preRateSamples)) + + d.ingestionRate.Tick() + + for _, push := range testData.pushes { + var request *cortexpbv2.WriteRequest + if enableHistogram { + request = makeWriteRequestV2WithHistogram(0, push.samples, push.metadata) + } else { + request = makeWriteRequestV2WithSamples(0, push.samples, push.metadata) + } + _, err := d.PushV2(ctx, request) + + if push.expectedError == nil { + assert.Nil(t, err) + } else { + assert.Equal(t, push.expectedError, err) + } + + d.ingestionRate.Tick() + + if testData.expectedMetrics != "" { + assert.NoError(t, testutil.GatherAndCompare(regs[0], strings.NewReader(testData.expectedMetrics), testData.metricNames...)) + } + } + }) + } + } +} + +func TestDistributorPRW2_PushQuery(t *testing.T) { + t.Parallel() + const shuffleShardSize = 5 + + ctx := user.InjectOrgID(context.Background(), "user") + nameMatcher := mustEqualMatcher(model.MetricNameLabel, "foo") + barMatcher := mustEqualMatcher("bar", "baz") + + type testcase struct { + name string + numIngesters int + happyIngesters int + samples int + metadata int + matchers []*labels.Matcher + expectedIngesters int + expectedResponse model.Matrix + expectedError error + shardByAllLabels bool + shuffleShardEnabled bool + } + + // We'll programmatically build the test cases now, as we want complete + // coverage along quite a few different axis. + testcases := []testcase{} + + // Run every test in both sharding modes. + for _, shardByAllLabels := range []bool{true, false} { + + // Test with between 2 and 10 ingesters. + for numIngesters := 2; numIngesters < 10; numIngesters++ { + + // Test with between 0 and numIngesters "happy" ingesters. + for happyIngesters := 0; happyIngesters <= numIngesters; happyIngesters++ { + + // Test either with shuffle-sharding enabled or disabled. + for _, shuffleShardEnabled := range []bool{false, true} { + scenario := fmt.Sprintf("shardByAllLabels=%v, numIngester=%d, happyIngester=%d, shuffleSharding=%v)", shardByAllLabels, numIngesters, happyIngesters, shuffleShardEnabled) + + // The number of ingesters we expect to query depends whether shuffle sharding and/or + // shard by all labels are enabled. + var expectedIngesters int + if shuffleShardEnabled { + expectedIngesters = min(shuffleShardSize, numIngesters) + } else if shardByAllLabels { + expectedIngesters = numIngesters + } else { + expectedIngesters = 3 // Replication factor + } + + // When we're not sharding by metric name, queriers with more than one + // failed ingester should fail. + if shardByAllLabels && numIngesters-happyIngesters > 1 { + testcases = append(testcases, testcase{ + name: fmt.Sprintf("ExpectFail(%s)", scenario), + numIngesters: numIngesters, + happyIngesters: happyIngesters, + matchers: []*labels.Matcher{nameMatcher, barMatcher}, + expectedError: errFail, + shardByAllLabels: shardByAllLabels, + shuffleShardEnabled: shuffleShardEnabled, + }) + continue + } + + // When we have less ingesters than replication factor, any failed ingester + // will cause a failure. + if numIngesters < 3 && happyIngesters < 2 { + testcases = append(testcases, testcase{ + name: fmt.Sprintf("ExpectFail(%s)", scenario), + numIngesters: numIngesters, + happyIngesters: happyIngesters, + matchers: []*labels.Matcher{nameMatcher, barMatcher}, + expectedError: errFail, + shardByAllLabels: shardByAllLabels, + shuffleShardEnabled: shuffleShardEnabled, + }) + continue + } + + // If we're sharding by metric name and we have failed ingesters, we can't + // tell ahead of time if the query will succeed, as we don't know which + // ingesters will hold the results for the query. + if !shardByAllLabels && numIngesters-happyIngesters > 1 { + continue + } + + // Reading all the samples back should succeed. + testcases = append(testcases, testcase{ + name: fmt.Sprintf("ReadAll(%s)", scenario), + numIngesters: numIngesters, + happyIngesters: happyIngesters, + samples: 10, + matchers: []*labels.Matcher{nameMatcher, barMatcher}, + expectedResponse: expectedResponse(0, 10), + expectedIngesters: expectedIngesters, + shardByAllLabels: shardByAllLabels, + shuffleShardEnabled: shuffleShardEnabled, + }) + + // As should reading none of the samples back. + testcases = append(testcases, testcase{ + name: fmt.Sprintf("ReadNone(%s)", scenario), + numIngesters: numIngesters, + happyIngesters: happyIngesters, + samples: 10, + matchers: []*labels.Matcher{nameMatcher, mustEqualMatcher("not", "found")}, + expectedResponse: expectedResponse(0, 0), + expectedIngesters: expectedIngesters, + shardByAllLabels: shardByAllLabels, + shuffleShardEnabled: shuffleShardEnabled, + }) + + // And reading each sample individually. + for i := 0; i < 10; i++ { + testcases = append(testcases, testcase{ + name: fmt.Sprintf("ReadOne(%s, sample=%d)", scenario, i), + numIngesters: numIngesters, + happyIngesters: happyIngesters, + samples: 10, + matchers: []*labels.Matcher{nameMatcher, mustEqualMatcher("sample", strconv.Itoa(i))}, + expectedResponse: expectedResponse(i, i+1), + expectedIngesters: expectedIngesters, + shardByAllLabels: shardByAllLabels, + shuffleShardEnabled: shuffleShardEnabled, + }) + } + } + } + } + } + + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ds, ingesters, _, _ := prepare(t, prepConfig{ + numIngesters: tc.numIngesters, + happyIngesters: tc.happyIngesters, + numDistributors: 1, + shardByAllLabels: tc.shardByAllLabels, + shuffleShardEnabled: tc.shuffleShardEnabled, + shuffleShardSize: shuffleShardSize, + }) + + request := makeWriteRequestV2WithSamples(0, tc.samples, tc.metadata) + writeResponse, err := ds[0].PushV2(ctx, request) + assert.Equal(t, &cortexpbv2.WriteResponse{}, writeResponse) + assert.Nil(t, err) + + var response model.Matrix + series, err := ds[0].QueryStream(ctx, 0, 10, tc.matchers...) + assert.Equal(t, tc.expectedError, err) + + if series == nil { + response, err = chunkcompat.SeriesChunksToMatrix(0, 10, nil) + } else { + response, err = chunkcompat.SeriesChunksToMatrix(0, 10, series.Chunkseries) + } + assert.NoError(t, err) + assert.Equal(t, tc.expectedResponse.String(), response.String()) + + // Check how many ingesters have been queried. + // Due to the quorum the distributor could cancel the last request towards ingesters + // if all other ones are successful, so we're good either has been queried X or X-1 + // ingesters. + if tc.expectedError == nil { + assert.Contains(t, []int{tc.expectedIngesters, tc.expectedIngesters - 1}, countMockIngestersCalls(ingesters, "QueryStream")) + } + }) + } +} + +func TestDistributorPRW2_QueryStream_ShouldReturnErrorIfMaxChunksPerQueryLimitIsReached(t *testing.T) { + t.Parallel() + const maxChunksLimit = 30 // Chunks are duplicated due to replication factor. + + for _, histogram := range []bool{true, false} { + ctx := user.InjectOrgID(context.Background(), "user") + limits := &validation.Limits{} + flagext.DefaultValues(limits) + limits.MaxChunksPerQuery = maxChunksLimit + + // Prepare distributors. + ds, _, _, _ := prepare(t, prepConfig{ + numIngesters: 3, + happyIngesters: 3, + numDistributors: 1, + shardByAllLabels: true, + limits: limits, + }) + + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, 0, maxChunksLimit, 0)) + + // Push a number of series below the max chunks limit. Each series has 1 sample, + // so expect 1 chunk per series when querying back. + initialSeries := maxChunksLimit / 3 + var writeReqV2 *cortexpbv2.WriteRequest + if histogram { + writeReqV2 = makeWriteRequestV2WithHistogram(0, initialSeries, 0) + } else { + writeReqV2 = makeWriteRequestV2WithSamples(0, initialSeries, 0) + } + + writeRes, err := ds[0].PushV2(ctx, writeReqV2) + assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) + assert.Nil(t, err) + + allSeriesMatchers := []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), + } + + // Since the number of series (and thus chunks) is equal to the limit (but doesn't + // exceed it), we expect a query running on all series to succeed. + queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + assert.Len(t, queryRes.Chunkseries, initialSeries) + + // Push more series to exceed the limit once we'll query back all series. + + for i := 0; i < maxChunksLimit; i++ { + writeReq := &cortexpbv2.WriteRequest{} + writeReq.Symbols = []string{"", "__name__", fmt.Sprintf("another_series_%d", i)} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: fmt.Sprintf("another_series_%d", i)}}, 0, 0, histogram, false), + ) + writeRes, err := ds[0].PushV2(ctx, writeReq) + assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) + assert.Nil(t, err) + } + + // Since the number of series (and thus chunks) is exceeding to the limit, we expect + // a query running on all series to fail. + _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.Error(t, err) + assert.Contains(t, err.Error(), "the query hit the max number of chunks limit") + } +} + +func TestDistributorPRW2_QueryStream_ShouldReturnErrorIfMaxSeriesPerQueryLimitIsReached(t *testing.T) { + t.Parallel() + const maxSeriesLimit = 10 + + for _, histogram := range []bool{true, false} { + ctx := user.InjectOrgID(context.Background(), "user") + limits := &validation.Limits{} + flagext.DefaultValues(limits) + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(maxSeriesLimit, 0, 0, 0)) + + // Prepare distributors. + ds, _, _, _ := prepare(t, prepConfig{ + numIngesters: 3, + happyIngesters: 3, + numDistributors: 1, + shardByAllLabels: true, + limits: limits, + }) + + // Push a number of series below the max series limit. + initialSeries := maxSeriesLimit + var writeReqV2 *cortexpbv2.WriteRequest + if histogram { + writeReqV2 = makeWriteRequestV2WithHistogram(0, initialSeries, 0) + } else { + writeReqV2 = makeWriteRequestV2WithSamples(0, initialSeries, 0) + } + + writeRes, err := ds[0].PushV2(ctx, writeReqV2) + assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) + assert.Nil(t, err) + + allSeriesMatchers := []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), + } + + // Since the number of series is equal to the limit (but doesn't + // exceed it), we expect a query running on all series to succeed. + queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + assert.Len(t, queryRes.Chunkseries, initialSeries) + + // Push more series to exceed the limit once we'll query back all series. + writeReq := &cortexpbv2.WriteRequest{} + writeReq.Symbols = []string{"", "__name__", "another_series"} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, 0, 0, histogram, false), + ) + + writeRes, err = ds[0].PushV2(ctx, writeReq) + assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) + assert.Nil(t, err) + + // Since the number of series is exceeding the limit, we expect + // a query running on all series to fail. + _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.Error(t, err) + assert.Contains(t, err.Error(), "max number of series limit") + } +} + +func TestDistributorPRW2_QueryStream_ShouldReturnErrorIfMaxChunkBytesPerQueryLimitIsReached(t *testing.T) { + t.Parallel() + const seriesToAdd = 10 + + for _, histogram := range []bool{true, false} { + ctx := user.InjectOrgID(context.Background(), "user") + limits := &validation.Limits{} + flagext.DefaultValues(limits) + + // Prepare distributors. + // Use replication factor of 2 to always read all the chunks from both ingesters, + // this guarantees us to always read the same chunks and have a stable test. + ds, _, _, _ := prepare(t, prepConfig{ + numIngesters: 2, + happyIngesters: 2, + numDistributors: 1, + shardByAllLabels: true, + limits: limits, + replicationFactor: 2, + }) + + allSeriesMatchers := []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), + } + // Push a single series to allow us to calculate the chunk size to calculate the limit for the test. + writeReq := &cortexpbv2.WriteRequest{} + writeReq.Symbols = []string{"", "__name__", "another_series"} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, 0, 0, histogram, false), + ) + writeRes, err := ds[0].PushV2(ctx, writeReq) + assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) + assert.Nil(t, err) + chunkSizeResponse, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + + // Use the resulting chunks size to calculate the limit as (series to add + our test series) * the response chunk size. + var responseChunkSize = chunkSizeResponse.ChunksSize() + var maxBytesLimit = (seriesToAdd) * responseChunkSize + + // Update the limiter with the calculated limits. + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, maxBytesLimit, 0, 0)) + + // Push a number of series below the max chunk bytes limit. Subtract one for the series added above. + var writeReqV2 *cortexpbv2.WriteRequest + if histogram { + writeReqV2 = makeWriteRequestV2WithHistogram(0, seriesToAdd-1, 0) + } else { + writeReqV2 = makeWriteRequestV2WithSamples(0, seriesToAdd-1, 0) + } + + writeRes, err = ds[0].PushV2(ctx, writeReqV2) + assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) + assert.Nil(t, err) + + // Since the number of chunk bytes is equal to the limit (but doesn't + // exceed it), we expect a query running on all series to succeed. + queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + assert.Len(t, queryRes.Chunkseries, seriesToAdd) + + // Push another series to exceed the chunk bytes limit once we'll query back all series. + writeReq = &cortexpbv2.WriteRequest{} + writeReq.Symbols = []string{"", "__name__", "another_series_1"} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series_1"}}, 0, 0, histogram, false), + ) + + writeRes, err = ds[0].PushV2(ctx, writeReq) + assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) + assert.Nil(t, err) + + // Since the aggregated chunk size is exceeding the limit, we expect + // a query running on all series to fail. + _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.Error(t, err) + assert.Equal(t, err, validation.LimitError(fmt.Sprintf(limiter.ErrMaxChunkBytesHit, maxBytesLimit))) + } +} + +func TestDistributorPRW2_QueryStream_ShouldReturnErrorIfMaxDataBytesPerQueryLimitIsReached(t *testing.T) { + t.Parallel() + const seriesToAdd = 10 + + for _, histogram := range []bool{true, false} { + ctx := user.InjectOrgID(context.Background(), "user") + limits := &validation.Limits{} + flagext.DefaultValues(limits) + + // Prepare distributors. + // Use replication factor of 2 to always read all the chunks from both ingesters, + // this guarantees us to always read the same chunks and have a stable test. + ds, _, _, _ := prepare(t, prepConfig{ + numIngesters: 2, + happyIngesters: 2, + numDistributors: 1, + shardByAllLabels: true, + limits: limits, + replicationFactor: 2, + }) + + allSeriesMatchers := []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), + } + // Push a single series to allow us to calculate the label size to calculate the limit for the test. + writeReq := &cortexpbv2.WriteRequest{} + writeReq.Symbols = []string{"", "__name__", "another_series"} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, 0, 0, histogram, false), + ) + + writeRes, err := ds[0].PushV2(ctx, writeReq) + assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) + assert.Nil(t, err) + dataSizeResponse, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + + // Use the resulting chunks size to calculate the limit as (series to add + our test series) * the response chunk size. + var dataSize = dataSizeResponse.Size() + var maxBytesLimit = (seriesToAdd) * dataSize * 2 // Multiplying by RF because the limit is applied before de-duping. + + // Update the limiter with the calculated limits. + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, 0, 0, maxBytesLimit)) + + // Push a number of series below the max chunk bytes limit. Subtract one for the series added above. + var writeReqV2 *cortexpbv2.WriteRequest + if histogram { + writeReqV2 = makeWriteRequestV2WithHistogram(0, seriesToAdd-1, 0) + } else { + writeReqV2 = makeWriteRequestV2WithSamples(0, seriesToAdd-1, 0) + } + + writeRes, err = ds[0].PushV2(ctx, writeReqV2) + assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) + assert.Nil(t, err) + + // Since the number of chunk bytes is equal to the limit (but doesn't + // exceed it), we expect a query running on all series to succeed. + queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + assert.Len(t, queryRes.Chunkseries, seriesToAdd) + + // Push another series to exceed the chunk bytes limit once we'll query back all series. + writeReq = &cortexpbv2.WriteRequest{} + writeReq.Symbols = []string{"", "__name__", "another_series_1"} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series_1"}}, 0, 0, histogram, false), + ) + + writeRes, err = ds[0].PushV2(ctx, writeReq) + assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) + assert.Nil(t, err) + + // Since the aggregated chunk size is exceeding the limit, we expect + // a query running on all series to fail. + _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.Error(t, err) + assert.Equal(t, err, validation.LimitError(fmt.Sprintf(limiter.ErrMaxDataBytesHit, maxBytesLimit))) + } +} + +func TestDistributorPRW2_Push_ShouldGuaranteeShardingTokenConsistencyOverTheTime(t *testing.T) { + t.Parallel() + ctx := user.InjectOrgID(context.Background(), "user") + tests := map[string]struct { + inputSeries labels.Labels + expectedSeries labels.Labels + expectedToken uint32 + }{ + "metric_1 with value_1": { + inputSeries: labels.Labels{ + {Name: "__name__", Value: "metric_1"}, + {Name: "cluster", Value: "cluster_1"}, + {Name: "key", Value: "value_1"}, + }, + expectedSeries: labels.Labels{ + {Name: "__name__", Value: "metric_1"}, + {Name: "cluster", Value: "cluster_1"}, + {Name: "key", Value: "value_1"}, + }, + expectedToken: 0xec0a2e9d, + }, + // TODO(Sungjin1212): Enable tests after implement PRW2 relabel + //"metric_1 with value_1 and dropped label due to config": { + // inputSeries: labels.Labels{ + // {Name: "__name__", Value: "metric_1"}, + // {Name: "cluster", Value: "cluster_1"}, + // {Name: "key", Value: "value_1"}, + // {Name: "dropped", Value: "unused"}, // will be dropped, doesn't need to be in correct order + // }, + // expectedSeries: labels.Labels{ + // {Name: "__name__", Value: "metric_1"}, + // {Name: "cluster", Value: "cluster_1"}, + // {Name: "key", Value: "value_1"}, + // }, + // expectedToken: 0xec0a2e9d, + //}, + //"metric_1 with value_1 and dropped HA replica label": { + // inputSeries: labels.Labels{ + // {Name: "__name__", Value: "metric_1"}, + // {Name: "cluster", Value: "cluster_1"}, + // {Name: "key", Value: "value_1"}, + // {Name: "__replica__", Value: "replica_1"}, + // }, + // expectedSeries: labels.Labels{ + // {Name: "__name__", Value: "metric_1"}, + // {Name: "cluster", Value: "cluster_1"}, + // {Name: "key", Value: "value_1"}, + // }, + // expectedToken: 0xec0a2e9d, + //}, + "metric_2 with value_1": { + inputSeries: labels.Labels{ + {Name: "__name__", Value: "metric_2"}, + {Name: "key", Value: "value_1"}, + }, + expectedSeries: labels.Labels{ + {Name: "__name__", Value: "metric_2"}, + {Name: "key", Value: "value_1"}, + }, + expectedToken: 0xa60906f2, + }, + "metric_1 with value_2": { + inputSeries: labels.Labels{ + {Name: "__name__", Value: "metric_1"}, + {Name: "key", Value: "value_2"}, + }, + expectedSeries: labels.Labels{ + {Name: "__name__", Value: "metric_1"}, + {Name: "key", Value: "value_2"}, + }, + expectedToken: 0x18abc8a2, + }, + } + + var limits validation.Limits + flagext.DefaultValues(&limits) + limits.DropLabels = []string{"dropped"} + limits.AcceptHASamples = true + + for testName, testData := range tests { + testData := testData + t.Run(testName, func(t *testing.T) { + t.Parallel() + ds, ingesters, _, _ := prepare(t, prepConfig{ + numIngesters: 2, + happyIngesters: 2, + numDistributors: 1, + shardByAllLabels: true, + limits: &limits, + }) + + // Push the series to the distributor + req := mockWriteRequestV2([]labels.Labels{testData.inputSeries}, 1, 1, false) + _, err := ds[0].PushV2(ctx, req) + require.NoError(t, err) + + // Since each test pushes only 1 series, we do expect the ingester + // to have received exactly 1 series + for i := range ingesters { + timeseries := ingesters[i].series() + assert.Equal(t, 1, len(timeseries)) + + series, ok := timeseries[testData.expectedToken] + require.True(t, ok) + assert.Equal(t, testData.expectedSeries, cortexpb.FromLabelAdaptersToLabels(series.Labels)) + } + }) + } +} + +func makeWriteRequestV2WithSamples(startTimestampMs int64, samples int, metadata int) *cortexpbv2.WriteRequest { + request := &cortexpbv2.WriteRequest{} + st := writev2.NewSymbolTable() + st.Symbolize("__name__") + st.Symbolize("foo") + st.Symbolize("bar") + st.Symbolize("baz") + + for i := 0; i < samples; i++ { + st.Symbolize("sample") + st.Symbolize(fmt.Sprintf("%d", i)) + request.Timeseries = append(request.Timeseries, makeTimeseriesV2FromST( + []cortexpb.LabelAdapter{ + {Name: model.MetricNameLabel, Value: "foo"}, + {Name: "bar", Value: "baz"}, + {Name: "sample", Value: fmt.Sprintf("%d", i)}, + }, &st, startTimestampMs+int64(i), i, false, i < metadata)) + } + + for i := 0; i < metadata-samples; i++ { + request.Timeseries = append(request.Timeseries, makeMetadataV2FromST(i, &st)) + } + + request.Symbols = st.Symbols() + + return request +} + +func TestDistributorPRW2_Push_LabelNameValidation(t *testing.T) { + t.Parallel() + inputLabels := labels.Labels{ + {Name: model.MetricNameLabel, Value: "foo"}, + {Name: "999.illegal", Value: "baz"}, + } + ctx := user.InjectOrgID(context.Background(), "user") + + tests := map[string]struct { + inputLabels labels.Labels + skipLabelNameValidationCfg bool + skipLabelNameValidationReq bool + errExpected bool + errMessage string + }{ + "label name validation is on by default": { + inputLabels: inputLabels, + errExpected: true, + errMessage: `sample invalid label: "999.illegal" metric "foo{999.illegal=\"baz\"}"`, + }, + "label name validation can be skipped via config": { + inputLabels: inputLabels, + skipLabelNameValidationCfg: true, + errExpected: false, + }, + "label name validation can be skipped via WriteRequest parameter": { + inputLabels: inputLabels, + skipLabelNameValidationReq: true, + errExpected: false, + }, + } + + for testName, tc := range tests { + tc := tc + for _, histogram := range []bool{true, false} { + histogram := histogram + t.Run(fmt.Sprintf("%s, histogram=%s", testName, strconv.FormatBool(histogram)), func(t *testing.T) { + t.Parallel() + ds, _, _, _ := prepare(t, prepConfig{ + numIngesters: 2, + happyIngesters: 2, + numDistributors: 1, + shuffleShardSize: 1, + skipLabelNameValidation: tc.skipLabelNameValidationCfg, + }) + req := mockWriteRequestV2([]labels.Labels{tc.inputLabels}, 42, 100000, histogram) + req.SkipLabelNameValidation = tc.skipLabelNameValidationReq + _, err := ds[0].PushV2(ctx, req) + if tc.errExpected { + fromError, _ := status.FromError(err) + assert.Equal(t, tc.errMessage, fromError.Message()) + } else { + assert.Nil(t, err) + } + }) + } + } +} + +func TestDistributorPRW2_Push_ExemplarValidation(t *testing.T) { + t.Parallel() + ctx := user.InjectOrgID(context.Background(), "user") + manyLabels := []string{model.MetricNameLabel, "test"} + for i := 1; i < 31; i++ { + manyLabels = append(manyLabels, fmt.Sprintf("name_%d", i), fmt.Sprintf("value_%d", i)) + } + + tests := map[string]struct { + req *cortexpbv2.WriteRequest + errMsg string + }{ + "valid exemplar": { + req: makeWriteRequestV2Exemplar([]string{model.MetricNameLabel, "test"}, 1000, []string{"foo", "bar"}), + }, + "rejects exemplar with no labels": { + req: makeWriteRequestV2Exemplar([]string{model.MetricNameLabel, "test"}, 1000, []string{}), + errMsg: `exemplar missing labels, timestamp: 1000 series: {__name__="test"} labels: {}`, + }, + "rejects exemplar with no timestamp": { + req: makeWriteRequestV2Exemplar([]string{model.MetricNameLabel, "test"}, 0, []string{"foo", "bar"}), + errMsg: `exemplar missing timestamp, timestamp: 0 series: {__name__="test"} labels: {foo="bar"}`, + }, + "rejects exemplar with too long labelset": { + req: makeWriteRequestV2Exemplar([]string{model.MetricNameLabel, "test"}, 1000, []string{"foo", strings.Repeat("0", 126)}), + errMsg: fmt.Sprintf(`exemplar combined labelset exceeds 128 characters, timestamp: 1000 series: {__name__="test"} labels: {foo="%s"}`, strings.Repeat("0", 126)), + }, + "rejects exemplar with too many series labels": { + req: makeWriteRequestV2Exemplar(manyLabels, 0, nil), + errMsg: "series has too many labels", + }, + "rejects exemplar with duplicate series labels": { + req: makeWriteRequestV2Exemplar([]string{model.MetricNameLabel, "test", "foo", "bar", "foo", "bar"}, 0, nil), + errMsg: "duplicate label name", + }, + "rejects exemplar with empty series label name": { + req: makeWriteRequestV2Exemplar([]string{model.MetricNameLabel, "test", "", "bar"}, 0, nil), + errMsg: "invalid label", + }, + } + + for testName, tc := range tests { + tc := tc + t.Run(testName, func(t *testing.T) { + t.Parallel() + ds, _, _, _ := prepare(t, prepConfig{ + numIngesters: 2, + happyIngesters: 2, + numDistributors: 1, + shuffleShardSize: 1, + }) + _, err := ds[0].PushV2(ctx, tc.req) + if tc.errMsg != "" { + fromError, _ := status.FromError(err) + assert.Contains(t, fromError.Message(), tc.errMsg) + } else { + assert.Nil(t, err) + } + }) + } +} + +func TestDistributorPRW2_MetricsForLabelMatchers_SingleSlowIngester(t *testing.T) { + t.Parallel() + for _, histogram := range []bool{true, false} { + // Create distributor + ds, ing, _, _ := prepare(t, prepConfig{ + numIngesters: 3, + happyIngesters: 3, + numDistributors: 1, + shardByAllLabels: true, + shuffleShardEnabled: true, + shuffleShardSize: 3, + replicationFactor: 3, + }) + + ing[2].queryDelay = 50 * time.Millisecond + + ctx := user.InjectOrgID(context.Background(), "test") + + now := model.Now() + + for i := 0; i < 100; i++ { + req := mockWriteRequestV2([]labels.Labels{{{Name: labels.MetricName, Value: "test"}, {Name: "app", Value: "m"}, {Name: "uniq8", Value: strconv.Itoa(i)}}}, 1, now.Unix(), histogram) + _, err := ds[0].PushV2(ctx, req) + require.NoError(t, err) + } + + for i := 0; i < 50; i++ { + _, err := ds[0].MetricsForLabelMatchers(ctx, now, now, nil, mustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test")) + require.NoError(t, err) + } + } +} + +func TestDistributorPRW2_MetricsForLabelMatchers(t *testing.T) { + t.Parallel() + const numIngesters = 5 + + fixtures := []struct { + lbls labels.Labels + value int64 + timestamp int64 + }{ + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "200"}}, 1, 100000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "500"}}, 1, 110000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_2"}}, 2, 200000}, + // The two following series have the same FastFingerprint=e002a3a451262627 + {labels.Labels{{Name: labels.MetricName, Value: "fast_fingerprint_collision"}, {Name: "app", Value: "l"}, {Name: "uniq0", Value: "0"}, {Name: "uniq1", Value: "1"}}, 1, 300000}, + {labels.Labels{{Name: labels.MetricName, Value: "fast_fingerprint_collision"}, {Name: "app", Value: "m"}, {Name: "uniq0", Value: "1"}, {Name: "uniq1", Value: "1"}}, 1, 300000}, + } + + tests := map[string]struct { + shuffleShardEnabled bool + shuffleShardSize int + matchers []*labels.Matcher + expectedResult []model.Metric + expectedIngesters int + queryLimiter *limiter.QueryLimiter + expectedErr error + }{ + "should return an empty response if no metric match": { + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "unknown"), + }, + expectedResult: []model.Metric{}, + expectedIngesters: numIngesters, + queryLimiter: limiter.NewQueryLimiter(0, 0, 0, 0), + expectedErr: nil, + }, + "should filter metrics by single matcher": { + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test_1"), + }, + expectedResult: []model.Metric{ + util.LabelsToMetric(fixtures[0].lbls), + util.LabelsToMetric(fixtures[1].lbls), + }, + expectedIngesters: numIngesters, + queryLimiter: limiter.NewQueryLimiter(0, 0, 0, 0), + expectedErr: nil, + }, + "should filter metrics by multiple matchers": { + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "status", "200"), + mustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test_1"), + }, + expectedResult: []model.Metric{ + util.LabelsToMetric(fixtures[0].lbls), + }, + expectedIngesters: numIngesters, + queryLimiter: limiter.NewQueryLimiter(0, 0, 0, 0), + expectedErr: nil, + }, + "should return all matching metrics even if their FastFingerprint collide": { + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "fast_fingerprint_collision"), + }, + expectedResult: []model.Metric{ + util.LabelsToMetric(fixtures[3].lbls), + util.LabelsToMetric(fixtures[4].lbls), + }, + expectedIngesters: numIngesters, + queryLimiter: limiter.NewQueryLimiter(0, 0, 0, 0), + expectedErr: nil, + }, + "should query only ingesters belonging to tenant's subring if shuffle sharding is enabled": { + shuffleShardEnabled: true, + shuffleShardSize: 3, + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test_1"), + }, + expectedResult: []model.Metric{ + util.LabelsToMetric(fixtures[0].lbls), + util.LabelsToMetric(fixtures[1].lbls), + }, + expectedIngesters: 3, + queryLimiter: limiter.NewQueryLimiter(0, 0, 0, 0), + expectedErr: nil, + }, + "should query all ingesters if shuffle sharding is enabled but shard size is 0": { + shuffleShardEnabled: true, + shuffleShardSize: 0, + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test_1"), + }, + expectedResult: []model.Metric{ + util.LabelsToMetric(fixtures[0].lbls), + util.LabelsToMetric(fixtures[1].lbls), + }, + expectedIngesters: numIngesters, + queryLimiter: limiter.NewQueryLimiter(0, 0, 0, 0), + expectedErr: nil, + }, + "should return err if series limit is exhausted": { + shuffleShardEnabled: true, + shuffleShardSize: 0, + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test_1"), + }, + expectedResult: nil, + expectedIngesters: numIngesters, + queryLimiter: limiter.NewQueryLimiter(1, 0, 0, 0), + expectedErr: validation.LimitError(fmt.Sprintf(limiter.ErrMaxSeriesHit, 1)), + }, + "should return err if data bytes limit is exhausted": { + shuffleShardEnabled: true, + shuffleShardSize: 0, + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test_1"), + }, + expectedResult: nil, + expectedIngesters: numIngesters, + queryLimiter: limiter.NewQueryLimiter(0, 0, 0, 1), + expectedErr: validation.LimitError(fmt.Sprintf(limiter.ErrMaxDataBytesHit, 1)), + }, + "should not exhaust series limit when only one series is fetched": { + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test_2"), + }, + expectedResult: []model.Metric{ + util.LabelsToMetric(fixtures[2].lbls), + }, + expectedIngesters: numIngesters, + queryLimiter: limiter.NewQueryLimiter(1, 0, 0, 0), + expectedErr: nil, + }, + } + + for testName, testData := range tests { + testData := testData + for _, histogram := range []bool{true, false} { + histogram := histogram + t.Run(fmt.Sprintf("%s, histogram=%s", testName, strconv.FormatBool(histogram)), func(t *testing.T) { + t.Parallel() + now := model.Now() + + // Create distributor + ds, ingesters, _, _ := prepare(t, prepConfig{ + numIngesters: numIngesters, + happyIngesters: numIngesters, + numDistributors: 1, + shardByAllLabels: true, + shuffleShardEnabled: testData.shuffleShardEnabled, + shuffleShardSize: testData.shuffleShardSize, + }) + + // Push fixtures + ctx := user.InjectOrgID(context.Background(), "test") + ctx = limiter.AddQueryLimiterToContext(ctx, testData.queryLimiter) + + for _, series := range fixtures { + req := mockWriteRequestV2([]labels.Labels{series.lbls}, series.value, series.timestamp, histogram) + _, err := ds[0].PushV2(ctx, req) + require.NoError(t, err) + } + + { + metrics, err := ds[0].MetricsForLabelMatchers(ctx, now, now, nil, testData.matchers...) + + if testData.expectedErr != nil { + assert.ErrorIs(t, err, testData.expectedErr) + return + } + + require.NoError(t, err) + assert.ElementsMatch(t, testData.expectedResult, metrics) + + // Check how many ingesters have been queried. + // Due to the quorum the distributor could cancel the last request towards ingesters + // if all other ones are successful, so we're good either has been queried X or X-1 + // ingesters. + assert.Contains(t, []int{testData.expectedIngesters, testData.expectedIngesters - 1}, countMockIngestersCalls(ingesters, "MetricsForLabelMatchers")) + } + + { + metrics, err := ds[0].MetricsForLabelMatchersStream(ctx, now, now, nil, testData.matchers...) + if testData.expectedErr != nil { + assert.ErrorIs(t, err, testData.expectedErr) + return + } + + require.NoError(t, err) + assert.ElementsMatch(t, testData.expectedResult, metrics) + + assert.Contains(t, []int{testData.expectedIngesters, testData.expectedIngesters - 1}, countMockIngestersCalls(ingesters, "MetricsForLabelMatchersStream")) + } + }) + } + } +} + +func BenchmarkDistributorPRW2_MetricsForLabelMatchers(b *testing.B) { + const ( + numIngesters = 100 + numSeriesPerRequest = 100 + ) + + tests := map[string]struct { + prepareConfig func(limits *validation.Limits) + prepareSeries func() ([]labels.Labels, []cortexpbv2.Sample) + matchers []*labels.Matcher + queryLimiter *limiter.QueryLimiter + expectedErr error + }{ + "get series within limits": { + prepareConfig: func(limits *validation.Limits) {}, + prepareSeries: func() ([]labels.Labels, []cortexpbv2.Sample) { + metrics := make([]labels.Labels, numSeriesPerRequest) + samples := make([]cortexpbv2.Sample, numSeriesPerRequest) + + for i := 0; i < numSeriesPerRequest; i++ { + lbls := labels.NewBuilder(labels.Labels{{Name: model.MetricNameLabel, Value: fmt.Sprintf("foo_%d", i)}}) + for i := 0; i < 10; i++ { + lbls.Set(fmt.Sprintf("name_%d", i), fmt.Sprintf("value_%d", i)) + } + + metrics[i] = lbls.Labels() + samples[i] = cortexpbv2.Sample{ + Value: float64(i), + Timestamp: time.Now().UnixNano() / int64(time.Millisecond), + } + } + + return metrics, samples + }, + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, "foo.+"), + }, + queryLimiter: limiter.NewQueryLimiter(100, 0, 0, 0), + expectedErr: nil, + }, + } + + for testName, testData := range tests { + b.Run(testName, func(b *testing.B) { + // Create distributor + ds, ingesters, _, _ := prepare(b, prepConfig{ + numIngesters: numIngesters, + happyIngesters: numIngesters, + numDistributors: 1, + shardByAllLabels: true, + shuffleShardEnabled: false, + shuffleShardSize: 0, + }) + + // Push fixtures + ctx := user.InjectOrgID(context.Background(), "test") + ctx = limiter.AddQueryLimiterToContext(ctx, testData.queryLimiter) + + // Prepare the series to remote write before starting the benchmark. + metrics, samples := testData.prepareSeries() + + if _, err := ds[0].PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, samples, nil, nil, cortexpbv2.API)); err != nil { + b.Fatalf("error pushing to distributor %v", err) + } + + // Run the benchmark. + b.ReportAllocs() + b.ResetTimer() + + for n := 0; n < b.N; n++ { + now := model.Now() + metrics, err := ds[0].MetricsForLabelMatchers(ctx, now, now, nil, testData.matchers...) + + if testData.expectedErr != nil { + assert.EqualError(b, err, testData.expectedErr.Error()) + return + } + + require.NoError(b, err) + + // Check how many ingesters have been queried. + // Due to the quorum the distributor could cancel the last request towards ingesters + // if all other ones are successful, so we're good either has been queried X or X-1 + // ingesters. + assert.Contains(b, []int{numIngesters, numIngesters - 1}, countMockIngestersCalls(ingesters, "MetricsForLabelMatchers")) + assert.Equal(b, numSeriesPerRequest, len(metrics)) + } + }) + } +} + +func TestDistributorPRW2_MetricsMetadata(t *testing.T) { + t.Parallel() + const numIngesters = 5 + + tests := map[string]struct { + shuffleShardEnabled bool + shuffleShardSize int + expectedIngesters int + }{ + "should query all ingesters if shuffle sharding is disabled": { + shuffleShardEnabled: false, + expectedIngesters: numIngesters, + }, + "should query all ingesters if shuffle sharding is enabled but shard size is 0": { + shuffleShardEnabled: true, + shuffleShardSize: 0, + expectedIngesters: numIngesters, + }, + "should query only ingesters belonging to tenant's subring if shuffle sharding is enabled": { + shuffleShardEnabled: true, + shuffleShardSize: 3, + expectedIngesters: 3, + }, + } + + for testName, testData := range tests { + testData := testData + t.Run(testName, func(t *testing.T) { + t.Parallel() + // Create distributor + ds, ingesters, _, _ := prepare(t, prepConfig{ + numIngesters: numIngesters, + happyIngesters: numIngesters, + numDistributors: 1, + shardByAllLabels: true, + shuffleShardEnabled: testData.shuffleShardEnabled, + shuffleShardSize: testData.shuffleShardSize, + limits: nil, + }) + + // Push metadata + ctx := user.InjectOrgID(context.Background(), "test") + + req := makeWriteRequestV2WithSamples(0, 0, 10) + _, err := ds[0].PushV2(ctx, req) + require.NoError(t, err) + + // Assert on metric metadata + metadata, err := ds[0].MetricsMetadata(ctx) + require.NoError(t, err) + assert.Equal(t, 10, len(metadata)) + + // Check how many ingesters have been queried. + // Due to the quorum the distributor could cancel the last request towards ingesters + // if all other ones are successful, so we're good either has been queried X or X-1 + // ingesters. + assert.Contains(t, []int{testData.expectedIngesters, testData.expectedIngesters - 1}, countMockIngestersCalls(ingesters, "MetricsMetadata")) + }) + } +} + +func makeWriteRequestV2WithHistogram(startTimestampMs int64, histogram int, metadata int) *cortexpbv2.WriteRequest { + request := &cortexpbv2.WriteRequest{} + st := writev2.NewSymbolTable() + st.Symbolize("__name__") + st.Symbolize("foo") + st.Symbolize("bar") + st.Symbolize("baz") + + for i := 0; i < histogram; i++ { + st.Symbolize("histogram") + st.Symbolize(fmt.Sprintf("%d", i)) + request.Timeseries = append(request.Timeseries, makeTimeseriesV2FromST( + []cortexpb.LabelAdapter{ + {Name: model.MetricNameLabel, Value: "foo"}, + {Name: "bar", Value: "baz"}, + {Name: "histogram", Value: fmt.Sprintf("%d", i)}, + }, &st, startTimestampMs+int64(i), i, true, i < metadata)) + } + + for i := 0; i < metadata-histogram; i++ { + request.Timeseries = append(request.Timeseries, makeMetadataV2FromST(i, &st)) + } + + request.Symbols = st.Symbols() + + return request +} + +func makeMetadataV2FromST(value int, st *writev2.SymbolsTable) cortexpbv2.TimeSeries { + t := cortexpbv2.TimeSeries{} + t.LabelsRefs = []uint32{1, 2} + helpRef := st.Symbolize(fmt.Sprintf("a help for metric_%d", value)) + t.Metadata.Type = cortexpbv2.METRIC_TYPE_COUNTER + t.Metadata.HelpRef = helpRef + + return t +} + +func makeTimeseriesV2FromST(labels []cortexpb.LabelAdapter, st *writev2.SymbolsTable, ts int64, value int, histogram bool, metadata bool) cortexpbv2.TimeSeries { + var helpRef uint32 + if metadata { + helpRef = st.Symbolize(fmt.Sprintf("a help for metric_%d", value)) + } + + t := cortexpbv2.TimeSeries{} + t.LabelsRefs = cortexpbv2.GetLabelRefsFromLabelAdapters(st.Symbols(), labels) + if metadata { + t.Metadata.Type = cortexpbv2.METRIC_TYPE_COUNTER + t.Metadata.HelpRef = helpRef + } + + if histogram { + t.Histograms = append(t.Histograms, cortexpbv2.HistogramToHistogramProto(ts, tsdbutil.GenerateTestHistogram(value))) + } else { + t.Samples = append(t.Samples, cortexpbv2.Sample{ + Timestamp: ts, + Value: float64(value), + }) + } + + return t +} + +func makeWriteRequestV2Timeseries(labels []cortexpb.LabelAdapter, ts int64, value int, histogram bool, metadata bool) cortexpbv2.TimeSeries { + st := writev2.NewSymbolTable() + for _, lb := range labels { + st.Symbolize(lb.Name) + st.Symbolize(lb.Value) + } + + var helpRef uint32 + if metadata { + helpRef = st.Symbolize(fmt.Sprintf("a help for metric_%d", value)) + } + + t := cortexpbv2.TimeSeries{} + t.LabelsRefs = cortexpbv2.GetLabelRefsFromLabelAdapters(st.Symbols(), labels) + if metadata { + t.Metadata.Type = cortexpbv2.METRIC_TYPE_COUNTER + t.Metadata.HelpRef = helpRef + } + + if histogram { + t.Histograms = append(t.Histograms, cortexpbv2.HistogramToHistogramProto(ts, tsdbutil.GenerateTestHistogram(value))) + } else { + t.Samples = append(t.Samples, cortexpbv2.Sample{ + Timestamp: ts, + Value: float64(value), + }) + } + + return t +} + +func makeWriteRequestV2Exemplar(seriesLabels []string, timestamp int64, exemplarLabels []string) *cortexpbv2.WriteRequest { + st := writev2.NewSymbolTable() + for _, l := range seriesLabels { + st.Symbolize(l) + } + for _, l := range exemplarLabels { + st.Symbolize(l) + } + + return &cortexpbv2.WriteRequest{ + Symbols: st.Symbols(), + Timeseries: []cortexpbv2.TimeSeries{ + { + LabelsRefs: cortexpbv2.GetLabelRefsFromLabelAdapters(st.Symbols(), cortexpb.FromLabelsToLabelAdapters(labels.FromStrings(seriesLabels...))), + Exemplars: []cortexpbv2.Exemplar{ + { + LabelsRefs: cortexpbv2.GetLabelRefsFromLabelAdapters(st.Symbols(), cortexpb.FromLabelsToLabelAdapters(labels.FromStrings(exemplarLabels...))), + Timestamp: timestamp, + }, + }, + }, + }, + } +} diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 6ecb27eb86b..3183cfd6166 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -14,7 +14,6 @@ import ( "testing" "time" - writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2" "google.golang.org/grpc/codes" "github.com/go-kit/log" @@ -1166,51 +1165,42 @@ func TestDistributor_PushQuery(t *testing.T) { for _, tc := range testcases { tc := tc - for _, isV2 := range []bool{false, true} { - t.Run(fmt.Sprintf("%s, isV2=%s", tc.name, strconv.FormatBool(isV2)), func(t *testing.T) { - t.Parallel() - ds, ingesters, _, _ := prepare(t, prepConfig{ - numIngesters: tc.numIngesters, - happyIngesters: tc.happyIngesters, - numDistributors: 1, - shardByAllLabels: tc.shardByAllLabels, - shuffleShardEnabled: tc.shuffleShardEnabled, - shuffleShardSize: shuffleShardSize, - }) + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ds, ingesters, _, _ := prepare(t, prepConfig{ + numIngesters: tc.numIngesters, + happyIngesters: tc.happyIngesters, + numDistributors: 1, + shardByAllLabels: tc.shardByAllLabels, + shuffleShardEnabled: tc.shuffleShardEnabled, + shuffleShardSize: shuffleShardSize, + }) - if isV2 { - request := makeWriteRequestV2(0, tc.samples, 0) - writeResponse, err := ds[0].PushV2(ctx, request) - assert.Equal(t, &cortexpbv2.WriteResponse{}, writeResponse) - assert.Nil(t, err) - } else { - request := makeWriteRequest(0, tc.samples, tc.metadata, 0) - writeResponse, err := ds[0].Push(ctx, request) - assert.Equal(t, &cortexpb.WriteResponse{}, writeResponse) - assert.Nil(t, err) - } + request := makeWriteRequest(0, tc.samples, tc.metadata, 0) + writeResponse, err := ds[0].Push(ctx, request) + assert.Equal(t, &cortexpb.WriteResponse{}, writeResponse) + assert.Nil(t, err) - var response model.Matrix - series, err := ds[0].QueryStream(ctx, 0, 10, tc.matchers...) - assert.Equal(t, tc.expectedError, err) + var response model.Matrix + series, err := ds[0].QueryStream(ctx, 0, 10, tc.matchers...) + assert.Equal(t, tc.expectedError, err) - if series == nil { - response, err = chunkcompat.SeriesChunksToMatrix(0, 10, nil) - } else { - response, err = chunkcompat.SeriesChunksToMatrix(0, 10, series.Chunkseries) - } - assert.NoError(t, err) - assert.Equal(t, tc.expectedResponse.String(), response.String()) + if series == nil { + response, err = chunkcompat.SeriesChunksToMatrix(0, 10, nil) + } else { + response, err = chunkcompat.SeriesChunksToMatrix(0, 10, series.Chunkseries) + } + assert.NoError(t, err) + assert.Equal(t, tc.expectedResponse.String(), response.String()) - // Check how many ingesters have been queried. - // Due to the quorum the distributor could cancel the last request towards ingesters - // if all other ones are successful, so we're good either has been queried X or X-1 - // ingesters. - if tc.expectedError == nil { - assert.Contains(t, []int{tc.expectedIngesters, tc.expectedIngesters - 1}, countMockIngestersCalls(ingesters, "QueryStream")) - } - }) - } + // Check how many ingesters have been queried. + // Due to the quorum the distributor could cancel the last request towards ingesters + // if all other ones are successful, so we're good either has been queried X or X-1 + // ingesters. + if tc.expectedError == nil { + assert.Contains(t, []int{tc.expectedIngesters, tc.expectedIngesters - 1}, countMockIngestersCalls(ingesters, "QueryStream")) + } + }) } } @@ -1218,557 +1208,277 @@ func TestDistributor_QueryStream_ShouldReturnErrorIfMaxChunksPerQueryLimitIsReac t.Parallel() const maxChunksLimit = 30 // Chunks are duplicated due to replication factor. - t.Run("Test V1", func(t *testing.T) { - for _, histogram := range []bool{true, false} { - ctx := user.InjectOrgID(context.Background(), "user") - limits := &validation.Limits{} - flagext.DefaultValues(limits) - limits.MaxChunksPerQuery = maxChunksLimit - - // Prepare distributors. - ds, _, _, _ := prepare(t, prepConfig{ - numIngesters: 3, - happyIngesters: 3, - numDistributors: 1, - shardByAllLabels: true, - limits: limits, - }) - - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, 0, maxChunksLimit, 0)) - - // Push a number of series below the max chunks limit. Each series has 1 sample, - // so expect 1 chunk per series when querying back. - initialSeries := maxChunksLimit / 3 - var writeReq *cortexpb.WriteRequest - if histogram { - writeReq = makeWriteRequest(0, 0, 0, initialSeries) - } else { - writeReq = makeWriteRequest(0, initialSeries, 0, 0) - } - writeRes, err := ds[0].Push(ctx, writeReq) - assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) - assert.Nil(t, err) - - allSeriesMatchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), - } + for _, histogram := range []bool{true, false} { + ctx := user.InjectOrgID(context.Background(), "user") + limits := &validation.Limits{} + flagext.DefaultValues(limits) + limits.MaxChunksPerQuery = maxChunksLimit - // Since the number of series (and thus chunks) is equal to the limit (but doesn't - // exceed it), we expect a query running on all series to succeed. - queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - assert.Len(t, queryRes.Chunkseries, initialSeries) - - // Push more series to exceed the limit once we'll query back all series. - writeReq = &cortexpb.WriteRequest{} - for i := 0; i < maxChunksLimit; i++ { - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: fmt.Sprintf("another_series_%d", i)}}, 0, 0, histogram), - ) - } + // Prepare distributors. + ds, _, _, _ := prepare(t, prepConfig{ + numIngesters: 3, + happyIngesters: 3, + numDistributors: 1, + shardByAllLabels: true, + limits: limits, + }) - writeRes, err = ds[0].Push(ctx, writeReq) - assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) - assert.Nil(t, err) + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, 0, maxChunksLimit, 0)) - // Since the number of series (and thus chunks) is exceeding to the limit, we expect - // a query running on all series to fail. - _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.Error(t, err) - assert.Contains(t, err.Error(), "the query hit the max number of chunks limit") + // Push a number of series below the max chunks limit. Each series has 1 sample, + // so expect 1 chunk per series when querying back. + initialSeries := maxChunksLimit / 3 + var writeReq *cortexpb.WriteRequest + if histogram { + writeReq = makeWriteRequest(0, 0, 0, initialSeries) + } else { + writeReq = makeWriteRequest(0, initialSeries, 0, 0) } - }) - t.Run("Test V2", func(t *testing.T) { - for _, histogram := range []bool{true, false} { - ctx := user.InjectOrgID(context.Background(), "user") - limits := &validation.Limits{} - flagext.DefaultValues(limits) - limits.MaxChunksPerQuery = maxChunksLimit - - // Prepare distributors. - ds, _, _, _ := prepare(t, prepConfig{ - numIngesters: 3, - happyIngesters: 3, - numDistributors: 1, - shardByAllLabels: true, - limits: limits, - }) - - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, 0, maxChunksLimit, 0)) + writeRes, err := ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) - // Push a number of series below the max chunks limit. Each series has 1 sample, - // so expect 1 chunk per series when querying back. - initialSeries := maxChunksLimit / 3 - var writeReqV2 *cortexpbv2.WriteRequest - if histogram { - writeReqV2 = makeWriteRequestV2(0, 0, initialSeries) - } else { - writeReqV2 = makeWriteRequestV2(0, initialSeries, 0) - } + allSeriesMatchers := []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), + } - writeRes, err := ds[0].PushV2(ctx, writeReqV2) - assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) - assert.Nil(t, err) + // Since the number of series (and thus chunks) is equal to the limit (but doesn't + // exceed it), we expect a query running on all series to succeed. + queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + assert.Len(t, queryRes.Chunkseries, initialSeries) - allSeriesMatchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), - } + // Push more series to exceed the limit once we'll query back all series. + writeReq = &cortexpb.WriteRequest{} + for i := 0; i < maxChunksLimit; i++ { + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: fmt.Sprintf("another_series_%d", i)}}, 0, 0, histogram), + ) + } - // Since the number of series (and thus chunks) is equal to the limit (but doesn't - // exceed it), we expect a query running on all series to succeed. - queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - assert.Len(t, queryRes.Chunkseries, initialSeries) - - // Push more series to exceed the limit once we'll query back all series. - - for i := 0; i < maxChunksLimit; i++ { - writeReq := &cortexpbv2.WriteRequest{} - writeReq.Symbols = []string{"", "__name__", fmt.Sprintf("another_series_%d", i)} - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: fmt.Sprintf("another_series_%d", i)}}, []string{"", "__name__", fmt.Sprintf("another_series_%d", i)}, 0, 0, histogram), - ) - writeRes, err := ds[0].PushV2(ctx, writeReq) - assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) - assert.Nil(t, err) - } + writeRes, err = ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) - // Since the number of series (and thus chunks) is exceeding to the limit, we expect - // a query running on all series to fail. - _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.Error(t, err) - assert.Contains(t, err.Error(), "the query hit the max number of chunks limit") - } - }) + // Since the number of series (and thus chunks) is exceeding to the limit, we expect + // a query running on all series to fail. + _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.Error(t, err) + assert.Contains(t, err.Error(), "the query hit the max number of chunks limit") + } } func TestDistributor_QueryStream_ShouldReturnErrorIfMaxSeriesPerQueryLimitIsReached(t *testing.T) { t.Parallel() const maxSeriesLimit = 10 - t.Run("Test V1", func(t *testing.T) { - for _, histogram := range []bool{true, false} { - ctx := user.InjectOrgID(context.Background(), "user") - limits := &validation.Limits{} - flagext.DefaultValues(limits) - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(maxSeriesLimit, 0, 0, 0)) - - // Prepare distributors. - ds, _, _, _ := prepare(t, prepConfig{ - numIngesters: 3, - happyIngesters: 3, - numDistributors: 1, - shardByAllLabels: true, - limits: limits, - }) - - // Push a number of series below the max series limit. - initialSeries := maxSeriesLimit - var writeReq *cortexpb.WriteRequest - if histogram { - writeReq = makeWriteRequest(0, 0, 0, initialSeries) - } else { - writeReq = makeWriteRequest(0, initialSeries, 0, 0) - } - - writeRes, err := ds[0].Push(ctx, writeReq) - assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) - assert.Nil(t, err) - - allSeriesMatchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), - } - - // Since the number of series is equal to the limit (but doesn't - // exceed it), we expect a query running on all series to succeed. - queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - assert.Len(t, queryRes.Chunkseries, initialSeries) - - // Push more series to exceed the limit once we'll query back all series. - writeReq = &cortexpb.WriteRequest{} - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, 0, 0, histogram), - ) + for _, histogram := range []bool{true, false} { + ctx := user.InjectOrgID(context.Background(), "user") + limits := &validation.Limits{} + flagext.DefaultValues(limits) + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(maxSeriesLimit, 0, 0, 0)) - writeRes, err = ds[0].Push(ctx, writeReq) - assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) - assert.Nil(t, err) + // Prepare distributors. + ds, _, _, _ := prepare(t, prepConfig{ + numIngesters: 3, + happyIngesters: 3, + numDistributors: 1, + shardByAllLabels: true, + limits: limits, + }) - // Since the number of series is exceeding the limit, we expect - // a query running on all series to fail. - _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.Error(t, err) - assert.Contains(t, err.Error(), "max number of series limit") + // Push a number of series below the max series limit. + initialSeries := maxSeriesLimit + var writeReq *cortexpb.WriteRequest + if histogram { + writeReq = makeWriteRequest(0, 0, 0, initialSeries) + } else { + writeReq = makeWriteRequest(0, initialSeries, 0, 0) } - }) - t.Run("Test V2", func(t *testing.T) { - for _, histogram := range []bool{true, false} { - ctx := user.InjectOrgID(context.Background(), "user") - limits := &validation.Limits{} - flagext.DefaultValues(limits) - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(maxSeriesLimit, 0, 0, 0)) - - // Prepare distributors. - ds, _, _, _ := prepare(t, prepConfig{ - numIngesters: 3, - happyIngesters: 3, - numDistributors: 1, - shardByAllLabels: true, - limits: limits, - }) - // Push a number of series below the max series limit. - initialSeries := maxSeriesLimit - var writeReqV2 *cortexpbv2.WriteRequest - if histogram { - writeReqV2 = makeWriteRequestV2(0, 0, initialSeries) - } else { - writeReqV2 = makeWriteRequestV2(0, initialSeries, 0) - } - - writeRes, err := ds[0].PushV2(ctx, writeReqV2) - assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) - assert.Nil(t, err) + writeRes, err := ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) - allSeriesMatchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), - } + allSeriesMatchers := []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), + } - // Since the number of series is equal to the limit (but doesn't - // exceed it), we expect a query running on all series to succeed. - queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - assert.Len(t, queryRes.Chunkseries, initialSeries) + // Since the number of series is equal to the limit (but doesn't + // exceed it), we expect a query running on all series to succeed. + queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + assert.Len(t, queryRes.Chunkseries, initialSeries) - // Push more series to exceed the limit once we'll query back all series. - writeReq := &cortexpbv2.WriteRequest{} - writeReq.Symbols = []string{"", "__name__", "another_series"} - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, []string{"", "__name__", "another_series"}, 0, 0, histogram), - ) + // Push more series to exceed the limit once we'll query back all series. + writeReq = &cortexpb.WriteRequest{} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, 0, 0, histogram), + ) - writeRes, err = ds[0].PushV2(ctx, writeReq) - assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) - assert.Nil(t, err) + writeRes, err = ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) - // Since the number of series is exceeding the limit, we expect - // a query running on all series to fail. - _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.Error(t, err) - assert.Contains(t, err.Error(), "max number of series limit") - } - }) + // Since the number of series is exceeding the limit, we expect + // a query running on all series to fail. + _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.Error(t, err) + assert.Contains(t, err.Error(), "max number of series limit") + } } func TestDistributor_QueryStream_ShouldReturnErrorIfMaxChunkBytesPerQueryLimitIsReached(t *testing.T) { t.Parallel() const seriesToAdd = 10 - t.Run("Test V1", func(t *testing.T) { - for _, histogram := range []bool{true, false} { - ctx := user.InjectOrgID(context.Background(), "user") - limits := &validation.Limits{} - flagext.DefaultValues(limits) - - // Prepare distributors. - // Use replication factor of 2 to always read all the chunks from both ingesters, - // this guarantees us to always read the same chunks and have a stable test. - ds, _, _, _ := prepare(t, prepConfig{ - numIngesters: 2, - happyIngesters: 2, - numDistributors: 1, - shardByAllLabels: true, - limits: limits, - replicationFactor: 2, - }) - - allSeriesMatchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), - } - // Push a single series to allow us to calculate the chunk size to calculate the limit for the test. - writeReq := &cortexpb.WriteRequest{} - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, 0, 0, histogram), - ) - writeRes, err := ds[0].Push(ctx, writeReq) - assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) - assert.Nil(t, err) - chunkSizeResponse, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - - // Use the resulting chunks size to calculate the limit as (series to add + our test series) * the response chunk size. - var responseChunkSize = chunkSizeResponse.ChunksSize() - var maxBytesLimit = (seriesToAdd) * responseChunkSize - - // Update the limiter with the calculated limits. - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, maxBytesLimit, 0, 0)) - - // Push a number of series below the max chunk bytes limit. Subtract one for the series added above. - if histogram { - writeReq = makeWriteRequest(0, 0, 0, seriesToAdd-1) - } else { - writeReq = makeWriteRequest(0, seriesToAdd-1, 0, 0) - } - writeRes, err = ds[0].Push(ctx, writeReq) - assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) - assert.Nil(t, err) - - // Since the number of chunk bytes is equal to the limit (but doesn't - // exceed it), we expect a query running on all series to succeed. - queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - assert.Len(t, queryRes.Chunkseries, seriesToAdd) - - // Push another series to exceed the chunk bytes limit once we'll query back all series. - writeReq = &cortexpb.WriteRequest{} - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series_1"}}, 0, 0, histogram), - ) - - writeRes, err = ds[0].Push(ctx, writeReq) - assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) - assert.Nil(t, err) + for _, histogram := range []bool{true, false} { + ctx := user.InjectOrgID(context.Background(), "user") + limits := &validation.Limits{} + flagext.DefaultValues(limits) + + // Prepare distributors. + // Use replication factor of 2 to always read all the chunks from both ingesters, + // this guarantees us to always read the same chunks and have a stable test. + ds, _, _, _ := prepare(t, prepConfig{ + numIngesters: 2, + happyIngesters: 2, + numDistributors: 1, + shardByAllLabels: true, + limits: limits, + replicationFactor: 2, + }) - // Since the aggregated chunk size is exceeding the limit, we expect - // a query running on all series to fail. - _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.Error(t, err) - assert.Equal(t, err, validation.LimitError(fmt.Sprintf(limiter.ErrMaxChunkBytesHit, maxBytesLimit))) + allSeriesMatchers := []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), } - }) - t.Run("Test V2", func(t *testing.T) { - for _, histogram := range []bool{true, false} { - ctx := user.InjectOrgID(context.Background(), "user") - limits := &validation.Limits{} - flagext.DefaultValues(limits) - - // Prepare distributors. - // Use replication factor of 2 to always read all the chunks from both ingesters, - // this guarantees us to always read the same chunks and have a stable test. - ds, _, _, _ := prepare(t, prepConfig{ - numIngesters: 2, - happyIngesters: 2, - numDistributors: 1, - shardByAllLabels: true, - limits: limits, - replicationFactor: 2, - }) - - allSeriesMatchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), - } - // Push a single series to allow us to calculate the chunk size to calculate the limit for the test. - writeReq := &cortexpbv2.WriteRequest{} - writeReq.Symbols = []string{"", "__name__", "another_series"} - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, []string{"", "__name__", "another_series"}, 0, 0, histogram), - ) - writeRes, err := ds[0].PushV2(ctx, writeReq) - assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) - assert.Nil(t, err) - chunkSizeResponse, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - - // Use the resulting chunks size to calculate the limit as (series to add + our test series) * the response chunk size. - var responseChunkSize = chunkSizeResponse.ChunksSize() - var maxBytesLimit = (seriesToAdd) * responseChunkSize + // Push a single series to allow us to calculate the chunk size to calculate the limit for the test. + writeReq := &cortexpb.WriteRequest{} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, 0, 0, histogram), + ) + writeRes, err := ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) + chunkSizeResponse, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) - // Update the limiter with the calculated limits. - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, maxBytesLimit, 0, 0)) + // Use the resulting chunks size to calculate the limit as (series to add + our test series) * the response chunk size. + var responseChunkSize = chunkSizeResponse.ChunksSize() + var maxBytesLimit = (seriesToAdd) * responseChunkSize - // Push a number of series below the max chunk bytes limit. Subtract one for the series added above. - var writeReqV2 *cortexpbv2.WriteRequest - if histogram { - writeReqV2 = makeWriteRequestV2(0, 0, seriesToAdd-1) - } else { - writeReqV2 = makeWriteRequestV2(0, seriesToAdd-1, 0) - } + // Update the limiter with the calculated limits. + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, maxBytesLimit, 0, 0)) - writeRes, err = ds[0].PushV2(ctx, writeReqV2) - assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) - assert.Nil(t, err) + // Push a number of series below the max chunk bytes limit. Subtract one for the series added above. + if histogram { + writeReq = makeWriteRequest(0, 0, 0, seriesToAdd-1) + } else { + writeReq = makeWriteRequest(0, seriesToAdd-1, 0, 0) + } + writeRes, err = ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) - // Since the number of chunk bytes is equal to the limit (but doesn't - // exceed it), we expect a query running on all series to succeed. - queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - assert.Len(t, queryRes.Chunkseries, seriesToAdd) + // Since the number of chunk bytes is equal to the limit (but doesn't + // exceed it), we expect a query running on all series to succeed. + queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + assert.Len(t, queryRes.Chunkseries, seriesToAdd) - // Push another series to exceed the chunk bytes limit once we'll query back all series. - writeReq = &cortexpbv2.WriteRequest{} - writeReq.Symbols = []string{"", "__name__", "another_series_1"} - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series_1"}}, []string{"", "__name__", "another_series_1"}, 0, 0, histogram), - ) + // Push another series to exceed the chunk bytes limit once we'll query back all series. + writeReq = &cortexpb.WriteRequest{} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series_1"}}, 0, 0, histogram), + ) - writeRes, err = ds[0].PushV2(ctx, writeReq) - assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) - assert.Nil(t, err) + writeRes, err = ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) - // Since the aggregated chunk size is exceeding the limit, we expect - // a query running on all series to fail. - _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.Error(t, err) - assert.Equal(t, err, validation.LimitError(fmt.Sprintf(limiter.ErrMaxChunkBytesHit, maxBytesLimit))) - } - }) + // Since the aggregated chunk size is exceeding the limit, we expect + // a query running on all series to fail. + _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.Error(t, err) + assert.Equal(t, err, validation.LimitError(fmt.Sprintf(limiter.ErrMaxChunkBytesHit, maxBytesLimit))) + } } func TestDistributor_QueryStream_ShouldReturnErrorIfMaxDataBytesPerQueryLimitIsReached(t *testing.T) { t.Parallel() const seriesToAdd = 10 - t.Run("Test V1", func(t *testing.T) { - for _, histogram := range []bool{true, false} { - ctx := user.InjectOrgID(context.Background(), "user") - limits := &validation.Limits{} - flagext.DefaultValues(limits) - - // Prepare distributors. - // Use replication factor of 2 to always read all the chunks from both ingesters, - // this guarantees us to always read the same chunks and have a stable test. - ds, _, _, _ := prepare(t, prepConfig{ - numIngesters: 2, - happyIngesters: 2, - numDistributors: 1, - shardByAllLabels: true, - limits: limits, - replicationFactor: 2, - }) - - allSeriesMatchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), - } - // Push a single series to allow us to calculate the label size to calculate the limit for the test. - writeReq := &cortexpb.WriteRequest{} - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, 0, 0, histogram), - ) - - writeRes, err := ds[0].Push(ctx, writeReq) - assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) - assert.Nil(t, err) - dataSizeResponse, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - - // Use the resulting chunks size to calculate the limit as (series to add + our test series) * the response chunk size. - var dataSize = dataSizeResponse.Size() - var maxBytesLimit = (seriesToAdd) * dataSize * 2 // Multiplying by RF because the limit is applied before de-duping. - - // Update the limiter with the calculated limits. - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, 0, 0, maxBytesLimit)) - - // Push a number of series below the max chunk bytes limit. Subtract one for the series added above. - if histogram { - writeReq = makeWriteRequest(0, 0, 0, seriesToAdd-1) - } else { - writeReq = makeWriteRequest(0, seriesToAdd-1, 0, 0) - } - writeRes, err = ds[0].Push(ctx, writeReq) - assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) - assert.Nil(t, err) - - // Since the number of chunk bytes is equal to the limit (but doesn't - // exceed it), we expect a query running on all series to succeed. - queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - assert.Len(t, queryRes.Chunkseries, seriesToAdd) - - // Push another series to exceed the chunk bytes limit once we'll query back all series. - writeReq = &cortexpb.WriteRequest{} - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series_1"}}, 0, 0, histogram), - ) - - writeRes, err = ds[0].Push(ctx, writeReq) - assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) - assert.Nil(t, err) + for _, histogram := range []bool{true, false} { + ctx := user.InjectOrgID(context.Background(), "user") + limits := &validation.Limits{} + flagext.DefaultValues(limits) + + // Prepare distributors. + // Use replication factor of 2 to always read all the chunks from both ingesters, + // this guarantees us to always read the same chunks and have a stable test. + ds, _, _, _ := prepare(t, prepConfig{ + numIngesters: 2, + happyIngesters: 2, + numDistributors: 1, + shardByAllLabels: true, + limits: limits, + replicationFactor: 2, + }) - // Since the aggregated chunk size is exceeding the limit, we expect - // a query running on all series to fail. - _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.Error(t, err) - assert.Equal(t, err, validation.LimitError(fmt.Sprintf(limiter.ErrMaxDataBytesHit, maxBytesLimit))) + allSeriesMatchers := []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), } - }) - t.Run("Test V2", func(t *testing.T) { - for _, histogram := range []bool{true, false} { - ctx := user.InjectOrgID(context.Background(), "user") - limits := &validation.Limits{} - flagext.DefaultValues(limits) - - // Prepare distributors. - // Use replication factor of 2 to always read all the chunks from both ingesters, - // this guarantees us to always read the same chunks and have a stable test. - ds, _, _, _ := prepare(t, prepConfig{ - numIngesters: 2, - happyIngesters: 2, - numDistributors: 1, - shardByAllLabels: true, - limits: limits, - replicationFactor: 2, - }) - - allSeriesMatchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), - } - // Push a single series to allow us to calculate the label size to calculate the limit for the test. - writeReq := &cortexpbv2.WriteRequest{} - writeReq.Symbols = []string{"", "__name__", "another_series"} - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, []string{"", "__name__", "another_series"}, 0, 0, histogram), - ) - - writeRes, err := ds[0].PushV2(ctx, writeReq) - assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) - assert.Nil(t, err) - dataSizeResponse, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - - // Use the resulting chunks size to calculate the limit as (series to add + our test series) * the response chunk size. - var dataSize = dataSizeResponse.Size() - var maxBytesLimit = (seriesToAdd) * dataSize * 2 // Multiplying by RF because the limit is applied before de-duping. + // Push a single series to allow us to calculate the label size to calculate the limit for the test. + writeReq := &cortexpb.WriteRequest{} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, 0, 0, histogram), + ) + + writeRes, err := ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) + dataSizeResponse, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) - // Update the limiter with the calculated limits. - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, 0, 0, maxBytesLimit)) + // Use the resulting chunks size to calculate the limit as (series to add + our test series) * the response chunk size. + var dataSize = dataSizeResponse.Size() + var maxBytesLimit = (seriesToAdd) * dataSize * 2 // Multiplying by RF because the limit is applied before de-duping. - // Push a number of series below the max chunk bytes limit. Subtract one for the series added above. - var writeReqV2 *cortexpbv2.WriteRequest - if histogram { - writeReqV2 = makeWriteRequestV2(0, 0, seriesToAdd-1) - } else { - writeReqV2 = makeWriteRequestV2(0, seriesToAdd-1, 0) - } + // Update the limiter with the calculated limits. + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, 0, 0, maxBytesLimit)) - writeRes, err = ds[0].PushV2(ctx, writeReqV2) - assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) - assert.Nil(t, err) + // Push a number of series below the max chunk bytes limit. Subtract one for the series added above. + if histogram { + writeReq = makeWriteRequest(0, 0, 0, seriesToAdd-1) + } else { + writeReq = makeWriteRequest(0, seriesToAdd-1, 0, 0) + } + writeRes, err = ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) - // Since the number of chunk bytes is equal to the limit (but doesn't - // exceed it), we expect a query running on all series to succeed. - queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - assert.Len(t, queryRes.Chunkseries, seriesToAdd) + // Since the number of chunk bytes is equal to the limit (but doesn't + // exceed it), we expect a query running on all series to succeed. + queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + assert.Len(t, queryRes.Chunkseries, seriesToAdd) - // Push another series to exceed the chunk bytes limit once we'll query back all series. - writeReq = &cortexpbv2.WriteRequest{} - writeReq.Symbols = []string{"", "__name__", "another_series_1"} - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestV2Timeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series_1"}}, []string{"", "__name__", "another_series_1"}, 0, 0, histogram), - ) + // Push another series to exceed the chunk bytes limit once we'll query back all series. + writeReq = &cortexpb.WriteRequest{} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series_1"}}, 0, 0, histogram), + ) - writeRes, err = ds[0].PushV2(ctx, writeReq) - assert.Equal(t, &cortexpbv2.WriteResponse{}, writeRes) - assert.Nil(t, err) + writeRes, err = ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) - // Since the aggregated chunk size is exceeding the limit, we expect - // a query running on all series to fail. - _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.Error(t, err) - assert.Equal(t, err, validation.LimitError(fmt.Sprintf(limiter.ErrMaxDataBytesHit, maxBytesLimit))) - } - }) + // Since the aggregated chunk size is exceeding the limit, we expect + // a query running on all series to fail. + _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.Error(t, err) + assert.Equal(t, err, validation.LimitError(fmt.Sprintf(limiter.ErrMaxDataBytesHit, maxBytesLimit))) + } } func TestDistributor_Push_LabelRemoval(t *testing.T) { @@ -2501,7 +2211,8 @@ func BenchmarkDistributor_Push(b *testing.B) { require.NoError(b, err) // Start the distributor. - distributor, err := New(distributorCfg, clientConfig, overrides, ingestersRing, true, nil, log.NewNopLogger()) + reg := prometheus.NewRegistry() + distributor, err := New(distributorCfg, clientConfig, overrides, ingestersRing, true, reg, log.NewNopLogger()) require.NoError(b, err) require.NoError(b, services.StartAndAwaitRunning(context.Background(), distributor)) @@ -2979,14 +2690,7 @@ func mockWriteRequestV2(lbls []labels.Labels, value int64, timestamp int64, hist } } - symbols := []string{""} - for _, lbl := range lbls { - lbl.Range(func(l labels.Label) { - symbols = append(symbols, l.Name, l.Value) - }) - } - - return cortexpbv2.ToWriteRequestV2(lbls, symbols, samples, histograms, nil, cortexpbv2.API) + return cortexpbv2.ToWriteRequestV2(lbls, samples, histograms, nil, cortexpbv2.API) } func mockWriteRequest(lbls []labels.Labels, value int64, timestampMs int64, histogram bool) *cortexpb.WriteRequest { @@ -3198,49 +2902,6 @@ func stopAll(ds []*Distributor, r *ring.Ring) { r.StopAsync() } -func makeWriteRequestV2(startTimestampMs int64, samples int, histograms int) *cortexpbv2.WriteRequest { - request := &cortexpbv2.WriteRequest{} - st := writev2.NewSymbolTable() - st.Symbolize("__name__") - st.Symbolize("foo") - st.Symbolize("bar") - st.Symbolize("baz") - st.Symbolize("sample") - st.Symbolize("histogram") - - //symbols := []string{"", "__name__", "foo", "bar", "baz", "sample"} - for i := 0; i < samples; i++ { - st.Symbolize(fmt.Sprintf("%d", i)) - //symbols = append(symbols, fmt.Sprintf("%d", i)) - } - //symbols = append(symbols, "histogram") - for i := 0; i < histograms; i++ { - st.Symbolize(fmt.Sprintf("%d", i)) - //symbols = append(symbols, fmt.Sprintf("%d", i)) - } - request.Symbols = st.Symbols() - - for i := 0; i < samples; i++ { - request.Timeseries = append(request.Timeseries, makeWriteRequestV2Timeseries( - []cortexpb.LabelAdapter{ - {Name: model.MetricNameLabel, Value: "foo"}, - {Name: "bar", Value: "baz"}, - {Name: "sample", Value: fmt.Sprintf("%d", i)}, - }, st.Symbols(), startTimestampMs+int64(i), i, false)) - } - - for i := 0; i < histograms; i++ { - request.Timeseries = append(request.Timeseries, makeWriteRequestV2Timeseries( - []cortexpb.LabelAdapter{ - {Name: model.MetricNameLabel, Value: "foo"}, - {Name: "bar", Value: "baz"}, - {Name: "histogram", Value: fmt.Sprintf("%d", i)}, - }, st.Symbols(), startTimestampMs+int64(i), i, true)) - } - - return request -} - func makeWriteRequest(startTimestampMs int64, samples int, metadata int, histograms int) *cortexpb.WriteRequest { request := &cortexpb.WriteRequest{} for i := 0; i < samples; i++ { @@ -3273,22 +2934,6 @@ func makeWriteRequest(startTimestampMs int64, samples int, metadata int, histogr return request } -func makeWriteRequestV2Timeseries(labels []cortexpb.LabelAdapter, symbols []string, ts int64, value int, histogram bool) cortexpbv2.TimeSeries { - t := cortexpbv2.TimeSeries{} - t.LabelsRefs = cortexpbv2.GetLabelRefsFromLabelAdapters(symbols, labels) - - if histogram { - t.Histograms = append(t.Histograms, cortexpbv2.HistogramToHistogramProto(ts, tsdbutil.GenerateTestHistogram(value))) - } else { - t.Samples = append(t.Samples, cortexpbv2.Sample{ - Timestamp: ts, - Value: float64(value), - }) - } - - return t -} - func makeWriteRequestTimeseries(labels []cortexpb.LabelAdapter, ts int64, value int, histogram bool) cortexpb.PreallocTimeseries { t := cortexpb.PreallocTimeseries{ TimeSeries: &cortexpb.TimeSeries{ @@ -3472,6 +3117,10 @@ func (i *mockIngester) PushV2(ctx context.Context, req *cortexpbv2.WriteRequest, i.timeseries = map[uint32]*cortexpb.PreallocTimeseries{} } + if i.metadata == nil { + i.metadata = map[uint32]map[cortexpb.MetricMetadata]struct{}{} + } + orgid, err := tenant.TenantID(ctx) if err != nil { return nil, err @@ -3481,7 +3130,8 @@ func (i *mockIngester) PushV2(ctx context.Context, req *cortexpbv2.WriteRequest, for j := range req.Timeseries { series := req.Timeseries[j] - labels := cortexpb.FromLabelsToLabelAdapters(series.ToLabels(&b, req.Symbols)) + tsLabels := series.ToLabels(&b, req.Symbols) + labels := cortexpb.FromLabelsToLabelAdapters(tsLabels) hash := shardByAllLabels(orgid, labels) existing, ok := i.timeseries[hash] var v1Sample []cortexpb.Sample @@ -3505,6 +3155,17 @@ func (i *mockIngester) PushV2(ctx context.Context, req *cortexpbv2.WriteRequest, } else { existing.Samples = append(existing.Samples, v1Sample...) } + + if series.Metadata.Type != cortexpbv2.METRIC_TYPE_UNSPECIFIED { + m := series.Metadata.ToV1Metadata(tsLabels.Get(model.MetricNameLabel), req.Symbols) + hash = shardByMetricName(orgid, m.MetricFamilyName) + set, ok := i.metadata[hash] + if !ok { + set = map[cortexpb.MetricMetadata]struct{}{} + i.metadata[hash] = set + } + set[*m] = struct{}{} + } } return &cortexpbv2.WriteResponse{}, nil diff --git a/pkg/distributor/stats.go b/pkg/distributor/stats.go index 04416087907..0f7fbc332d0 100644 --- a/pkg/distributor/stats.go +++ b/pkg/distributor/stats.go @@ -1,16 +1,16 @@ package distributor import ( - "sync/atomic" //lint:ignore faillint we can't use go.uber.org/atomic with a protobuf struct without wrapping it. + "go.uber.org/atomic" ) type WriteStats struct { // Samples represents X-Prometheus-Remote-Write-Written-Samples - Samples int64 + Samples atomic.Int64 // Histograms represents X-Prometheus-Remote-Write-Written-Histograms - Histograms int64 + Histograms atomic.Int64 // Exemplars represents X-Prometheus-Remote-Write-Written-Exemplars - Exemplars int64 + Exemplars atomic.Int64 } func (w *WriteStats) SetSamples(samples int64) { @@ -18,7 +18,7 @@ func (w *WriteStats) SetSamples(samples int64) { return } - atomic.StoreInt64(&w.Samples, samples) + w.Samples.Store(samples) } func (w *WriteStats) SetHistograms(histograms int64) { @@ -26,7 +26,7 @@ func (w *WriteStats) SetHistograms(histograms int64) { return } - atomic.StoreInt64(&w.Histograms, histograms) + w.Histograms.Store(histograms) } func (w *WriteStats) SetExemplars(exemplars int64) { @@ -34,7 +34,7 @@ func (w *WriteStats) SetExemplars(exemplars int64) { return } - atomic.StoreInt64(&w.Exemplars, exemplars) + w.Exemplars.Store(exemplars) } func (w *WriteStats) LoadSamples() int64 { @@ -42,7 +42,7 @@ func (w *WriteStats) LoadSamples() int64 { return 0 } - return atomic.LoadInt64(&w.Samples) + return w.Samples.Load() } func (w *WriteStats) LoadHistogram() int64 { @@ -50,7 +50,7 @@ func (w *WriteStats) LoadHistogram() int64 { return 0 } - return atomic.LoadInt64(&w.Histograms) + return w.Histograms.Load() } func (w *WriteStats) LoadExemplars() int64 { @@ -58,5 +58,5 @@ func (w *WriteStats) LoadExemplars() int64 { return 0 } - return atomic.LoadInt64(&w.Exemplars) + return w.Exemplars.Load() } diff --git a/pkg/distributor/stats_test.go b/pkg/distributor/stats_test.go new file mode 100644 index 00000000000..10f0bf87b2b --- /dev/null +++ b/pkg/distributor/stats_test.go @@ -0,0 +1,41 @@ +package distributor + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_SetAndLoad(t *testing.T) { + s := &WriteStats{} + + t.Run("Samples", func(t *testing.T) { + s.SetSamples(3) + assert.Equal(t, int64(3), s.LoadSamples()) + }) + t.Run("Histograms", func(t *testing.T) { + s.SetHistograms(10) + assert.Equal(t, int64(10), s.LoadHistogram()) + }) + t.Run("Exemplars", func(t *testing.T) { + s.SetExemplars(2) + assert.Equal(t, int64(2), s.LoadExemplars()) + }) +} + +func Test_NilReceiver(t *testing.T) { + var s *WriteStats + + t.Run("Samples", func(t *testing.T) { + s.SetSamples(3) + assert.Equal(t, int64(0), s.LoadSamples()) + }) + t.Run("Histograms", func(t *testing.T) { + s.SetHistograms(10) + assert.Equal(t, int64(0), s.LoadHistogram()) + }) + t.Run("Exemplars", func(t *testing.T) { + s.SetExemplars(2) + assert.Equal(t, int64(0), s.LoadExemplars()) + }) +} diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 0b92b0a5a0c..cb7fae7349b 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1320,11 +1320,14 @@ func (i *Ingester) PushV2(ctx context.Context, req *cortexpbv2.WriteRequest) (*c } } - if err := i.appendMetadata(userID, ts.Metadata.ToV1Metadata(tsLabels.Get(model.MetricNameLabel), req.Symbols)); err == nil { - succeededMetadataCount++ - } else { - level.Warn(i.logger).Log("msg", "failed to ingest metadata", "err", err) - failedMetadataCount++ + if ts.Metadata.Type != cortexpbv2.METRIC_TYPE_UNSPECIFIED { + metaData := ts.Metadata.ToV1Metadata(tsLabels.Get(model.MetricNameLabel), req.Symbols) + if err := i.appendMetadata(userID, metaData); err == nil { + succeededMetadataCount++ + } else { + level.Warn(i.logger).Log("msg", "failed to ingest metadata", "err", err) + failedMetadataCount++ + } } } // At this point all samples have been added to the appender, so we can track the time it took. diff --git a/pkg/ingester/ingester_prw2_test.go b/pkg/ingester/ingester_prw2_test.go index 4dbff35aedc..aee4e7e98a2 100644 --- a/pkg/ingester/ingester_prw2_test.go +++ b/pkg/ingester/ingester_prw2_test.go @@ -11,6 +11,8 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "sort" "strconv" "strings" @@ -53,7 +55,1364 @@ import ( "github.com/cortexproject/cortex/pkg/util/validation" ) -// TODO(Sungjin1212): Add TestIngester_Push, TestIngesterMetricLimitExceeded, TestIngesterUserLimitExceeded, TestIngesterPerLabelsetLimitExceeded +func TestIngesterPRW2_Push(t *testing.T) { + metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} + metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) + metricNames := []string{ + "cortex_ingester_ingested_samples_total", + "cortex_ingester_ingested_samples_failures_total", + "cortex_ingester_memory_series", + "cortex_ingester_memory_users", + "cortex_ingester_memory_series_created_total", + "cortex_ingester_memory_series_removed_total", + } + userID := "test" + + testHistogramV2 := cortexpbv2.HistogramToHistogramProto(10, tsdbutil.GenerateTestHistogram(1)) + testHistogram := cortexpb.HistogramToHistogramProto(10, tsdbutil.GenerateTestHistogram(1)) + testFloatHistogramV2 := cortexpbv2.FloatHistogramToHistogramProto(11, tsdbutil.GenerateTestFloatHistogram(1)) + testFloatHistogram := cortexpb.FloatHistogramToHistogramProto(11, tsdbutil.GenerateTestFloatHistogram(1)) + tests := map[string]struct { + reqs []*cortexpbv2.WriteRequest + expectedErr error + expectedIngested []cortexpb.TimeSeries + expectedMetadataIngested []*cortexpb.MetricMetadata + expectedExemplarsIngested []cortexpb.TimeSeries + expectedMetrics string + additionalMetrics []string + disableActiveSeries bool + maxExemplars int + oooTimeWindow time.Duration + disableNativeHistogram bool + }{ + "should record native histogram discarded": { + reqs: []*cortexpbv2.WriteRequest{ + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 2, Timestamp: 10}}, + []cortexpbv2.Histogram{{Timestamp: 10}}, + []cortexpbv2.Metadata{{Type: cortexpbv2.METRIC_TYPE_GAUGE, HelpRef: 3}}, + cortexpbv2.API, + "a help for metric_name_2"), + }, + expectedErr: nil, + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 10}}}, + }, + expectedMetadataIngested: []*cortexpb.MetricMetadata{ + {MetricFamilyName: "test", Help: "a help for metric_name_2", Unit: "", Type: cortexpb.GAUGE}, + }, + additionalMetrics: []string{"cortex_discarded_samples_total", "cortex_ingester_active_series"}, + disableNativeHistogram: true, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{reason="native-histogram-sample",user="test"} 1 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "should succeed on valid series and metadata": { + reqs: []*cortexpbv2.WriteRequest{ + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 1, Timestamp: 9}}, + nil, + []cortexpbv2.Metadata{{HelpRef: 3, Type: cortexpbv2.METRIC_TYPE_COUNTER}}, + cortexpbv2.API, + "a help for metric_name_1"), + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 2, Timestamp: 10}}, + nil, + []cortexpbv2.Metadata{{HelpRef: 3, Type: cortexpbv2.METRIC_TYPE_GAUGE}}, + cortexpbv2.API, + "a help for metric_name_2"), + }, + expectedErr: nil, + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 9}, {Value: 2, TimestampMs: 10}}}, + }, + expectedMetadataIngested: []*cortexpb.MetricMetadata{ + {MetricFamilyName: "test", Help: "a help for metric_name_2", Unit: "", Type: cortexpb.GAUGE}, + {MetricFamilyName: "test", Help: "a help for metric_name_1", Unit: "", Type: cortexpb.COUNTER}, + }, + additionalMetrics: []string{ + // Metadata. + "cortex_ingester_memory_metadata", + "cortex_ingester_memory_metadata_created_total", + "cortex_ingester_ingested_metadata_total", + "cortex_ingester_ingested_metadata_failures_total", + "cortex_ingester_active_series", + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_metadata_failures_total The total number of metadata that errored on ingestion. + # TYPE cortex_ingester_ingested_metadata_failures_total counter + cortex_ingester_ingested_metadata_failures_total 0 + # HELP cortex_ingester_ingested_metadata_total The total number of metadata ingested. + # TYPE cortex_ingester_ingested_metadata_total counter + cortex_ingester_ingested_metadata_total 2 + # HELP cortex_ingester_memory_metadata The current number of metadata in memory. + # TYPE cortex_ingester_memory_metadata gauge + cortex_ingester_memory_metadata 2 + # HELP cortex_ingester_memory_metadata_created_total The total number of metadata that were created per user + # TYPE cortex_ingester_memory_metadata_created_total counter + cortex_ingester_memory_metadata_created_total{user="test"} 2 + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 2 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "should succeed on valid series with exemplars": { + maxExemplars: 2, + reqs: []*cortexpbv2.WriteRequest{ + // Ingesting an exemplar requires a sample to create the series first + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 1, Timestamp: 9}}, + nil, + nil, + cortexpbv2.API), + { + Symbols: []string{"", "__name__", "test", "traceID", "123", "456"}, + Timeseries: []cortexpbv2.TimeSeries{ + { + LabelsRefs: []uint32{1, 2}, + Exemplars: []cortexpbv2.Exemplar{ + { + LabelsRefs: []uint32{3, 4}, + Timestamp: 1000, + Value: 1000, + }, + { + LabelsRefs: []uint32{3, 5}, + Timestamp: 1001, + Value: 1001, + }, + }, + }, + }, + }, + }, + expectedErr: nil, + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 9}}}, + }, + expectedExemplarsIngested: []cortexpb.TimeSeries{ + { + Labels: metricLabelAdapters, + Exemplars: []cortexpb.Exemplar{ + { + Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "123"}}, + TimestampMs: 1000, + Value: 1000, + }, + { + Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "456"}}, + TimestampMs: 1001, + Value: 1001, + }, + }, + }, + }, + expectedMetadataIngested: nil, + additionalMetrics: []string{ + "cortex_ingester_tsdb_exemplar_exemplars_appended_total", + "cortex_ingester_tsdb_exemplar_exemplars_in_storage", + "cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage", + "cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds", + "cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total", + "cortex_ingester_active_series", + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + + # HELP cortex_ingester_tsdb_exemplar_exemplars_appended_total Total number of TSDB exemplars appended. + # TYPE cortex_ingester_tsdb_exemplar_exemplars_appended_total counter + cortex_ingester_tsdb_exemplar_exemplars_appended_total 2 + + # HELP cortex_ingester_tsdb_exemplar_exemplars_in_storage Number of TSDB exemplars currently in storage. + # TYPE cortex_ingester_tsdb_exemplar_exemplars_in_storage gauge + cortex_ingester_tsdb_exemplar_exemplars_in_storage 2 + + # HELP cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage Number of TSDB series with exemplars currently in storage. + # TYPE cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage gauge + cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{user="test"} 1 + + # HELP cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds The timestamp of the oldest exemplar stored in circular storage. Useful to check for what time range the current exemplar buffer limit allows. This usually means the last timestamp for all exemplars for a typical setup. This is not true though if one of the series timestamp is in future compared to rest series. + # TYPE cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds gauge + cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds{user="test"} 1 + + # HELP cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total Total number of out of order exemplar ingestion failed attempts. + # TYPE cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total counter + cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total 0 + `, + }, + "successful push, active series disabled": { + disableActiveSeries: true, + reqs: []*cortexpbv2.WriteRequest{ + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 1, Timestamp: 9}}, + nil, + nil, + cortexpbv2.API), + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 2, Timestamp: 10}}, + nil, + nil, + cortexpbv2.API), + }, + expectedErr: nil, + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 9}, {Value: 2, TimestampMs: 10}}}, + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 2 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + `, + }, + "ooo disabled, should soft fail on sample out of order": { + reqs: []*cortexpbv2.WriteRequest{ + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 2, Timestamp: 10}}, + nil, + nil, + cortexpbv2.API), + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 1, Timestamp: 9}}, + []cortexpbv2.Histogram{ + cortexpbv2.HistogramToHistogramProto(9, tsdbutil.GenerateTestHistogram(1)), + }, + nil, + cortexpbv2.API), + }, + expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestErr(storage.ErrOutOfOrderSample, model.Time(9), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 10}}}, + }, + additionalMetrics: []string{ + "cortex_ingester_tsdb_out_of_order_samples_total", + "cortex_ingester_tsdb_head_out_of_order_samples_appended_total", + "cortex_discarded_samples_total", + "cortex_ingester_active_series", + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 2 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_tsdb_head_out_of_order_samples_appended_total Total number of appended out of order samples. + # TYPE cortex_ingester_tsdb_head_out_of_order_samples_appended_total counter + cortex_ingester_tsdb_head_out_of_order_samples_appended_total{type="float",user="test"} 0 + # HELP cortex_ingester_tsdb_out_of_order_samples_total Total number of out of order samples ingestion failed attempts due to out of order being disabled. + # TYPE cortex_ingester_tsdb_out_of_order_samples_total counter + cortex_ingester_tsdb_out_of_order_samples_total{type="float",user="test"} 1 + cortex_ingester_tsdb_out_of_order_samples_total{type="histogram",user="test"} 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{reason="sample-out-of-order",user="test"} 2 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "ooo disabled, should soft fail on sample out of bound": { + reqs: []*cortexpbv2.WriteRequest{ + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 2, Timestamp: 1575043969}}, + nil, + nil, + cortexpbv2.API), + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 1, Timestamp: 1575043969 - (86400 * 1000)}}, + []cortexpbv2.Histogram{ + cortexpbv2.HistogramToHistogramProto(1575043969-(86400*1000), tsdbutil.GenerateTestHistogram(1)), + }, + nil, + cortexpbv2.API), + }, + expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestErr(storage.ErrOutOfBounds, model.Time(1575043969-(86400*1000)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}}, + }, + additionalMetrics: []string{"cortex_ingester_active_series"}, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 2 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{reason="sample-out-of-bounds",user="test"} 2 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "ooo enabled, should soft fail on sample too old": { + reqs: []*cortexpbv2.WriteRequest{ + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 2, Timestamp: 1575043969}}, + nil, + nil, + cortexpbv2.API), + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 1, Timestamp: 1575043969 - (600 * 1000)}}, + nil, + nil, + cortexpbv2.API), + }, + oooTimeWindow: 5 * time.Minute, + expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestErr(storage.ErrTooOldSample, model.Time(1575043969-(600*1000)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}}, + }, + additionalMetrics: []string{ + "cortex_discarded_samples_total", + "cortex_ingester_active_series", + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 1 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{reason="sample-too-old",user="test"} 1 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "ooo enabled, should succeed": { + reqs: []*cortexpbv2.WriteRequest{ + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 2, Timestamp: 1575043969}}, + nil, + nil, + cortexpbv2.API), + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 1, Timestamp: 1575043969 - (60 * 1000)}}, + nil, + nil, + cortexpbv2.API), + }, + oooTimeWindow: 5 * time.Minute, + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 1575043969 - (60 * 1000)}, {Value: 2, TimestampMs: 1575043969}}}, + }, + additionalMetrics: []string{"cortex_ingester_active_series"}, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 2 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "should soft fail on two different sample values at the same timestamp": { + reqs: []*cortexpbv2.WriteRequest{ + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 2, Timestamp: 1575043969}}, + nil, + nil, + cortexpbv2.API), + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 1, Timestamp: 1575043969}}, + nil, + nil, + cortexpbv2.API), + }, + expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestErr(storage.NewDuplicateFloatErr(1575043969, 2, 1), model.Time(1575043969), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}}, + }, + additionalMetrics: []string{"cortex_discarded_samples_total", "cortex_ingester_active_series"}, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 1 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{reason="new-value-for-timestamp",user="test"} 1 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "should soft fail on exemplar with unknown series": { + maxExemplars: 1, + reqs: []*cortexpbv2.WriteRequest{ + // Ingesting an exemplar requires a sample to create the series first + // This is not done here. + { + Symbols: []string{"", "__name__", "test", "traceID", "123"}, + Timeseries: []cortexpbv2.TimeSeries{ + { + LabelsRefs: []uint32{1, 2}, + Exemplars: []cortexpbv2.Exemplar{ + { + LabelsRefs: []uint32{3, 4}, + Timestamp: 1000, + Value: 1000, + }, + }, + }, + }, + }, + }, + expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestExemplarErr(errExemplarRef, model.Time(1000), cortexpb.FromLabelsToLabelAdapters(metricLabels), []cortexpb.LabelAdapter{{Name: "traceID", Value: "123"}}), userID).Error()), + expectedIngested: nil, + expectedMetadataIngested: nil, + additionalMetrics: []string{ + "cortex_ingester_tsdb_exemplar_exemplars_appended_total", + "cortex_ingester_tsdb_exemplar_exemplars_in_storage", + "cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage", + "cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds", + "cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total", + "cortex_ingester_active_series", + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 0 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 0 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 0 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 0 + + # HELP cortex_ingester_tsdb_exemplar_exemplars_appended_total Total number of TSDB exemplars appended. + # TYPE cortex_ingester_tsdb_exemplar_exemplars_appended_total counter + cortex_ingester_tsdb_exemplar_exemplars_appended_total 0 + + # HELP cortex_ingester_tsdb_exemplar_exemplars_in_storage Number of TSDB exemplars currently in storage. + # TYPE cortex_ingester_tsdb_exemplar_exemplars_in_storage gauge + cortex_ingester_tsdb_exemplar_exemplars_in_storage 0 + + # HELP cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage Number of TSDB series with exemplars currently in storage. + # TYPE cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage gauge + cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{user="test"} 0 + + # HELP cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds The timestamp of the oldest exemplar stored in circular storage. Useful to check for what time range the current exemplar buffer limit allows. This usually means the last timestamp for all exemplars for a typical setup. This is not true though if one of the series timestamp is in future compared to rest series. + # TYPE cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds gauge + cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds{user="test"} 0 + + # HELP cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total Total number of out of order exemplar ingestion failed attempts. + # TYPE cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total counter + cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total 0 + `, + }, + "should succeed when only native histogram present if enabled": { + reqs: []*cortexpbv2.WriteRequest{ + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + nil, + []cortexpbv2.Histogram{testHistogramV2}, + nil, + cortexpbv2.API), + }, + expectedErr: nil, + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Histograms: []cortexpb.Histogram{testHistogram}}, + }, + additionalMetrics: []string{ + "cortex_ingester_tsdb_head_samples_appended_total", + "cortex_ingester_active_series", + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_tsdb_head_out_of_order_samples_appended_total Total number of appended out of order samples. + # TYPE cortex_ingester_tsdb_head_out_of_order_samples_appended_total counter + cortex_ingester_tsdb_head_out_of_order_samples_appended_total{type="float",user="test"} 0 + # HELP cortex_ingester_tsdb_head_samples_appended_total Total number of appended samples. + # TYPE cortex_ingester_tsdb_head_samples_appended_total counter + cortex_ingester_tsdb_head_samples_appended_total{type="float",user="test"} 0 + cortex_ingester_tsdb_head_samples_appended_total{type="histogram",user="test"} 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "should succeed when only float native histogram present if enabled": { + reqs: []*cortexpbv2.WriteRequest{ + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + nil, + []cortexpbv2.Histogram{testFloatHistogramV2}, + nil, + cortexpbv2.API), + }, + expectedErr: nil, + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Histograms: []cortexpb.Histogram{testFloatHistogram}}, + }, + additionalMetrics: []string{ + "cortex_ingester_tsdb_head_samples_appended_total", + "cortex_ingester_active_series", + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_tsdb_head_out_of_order_samples_appended_total Total number of appended out of order samples. + # TYPE cortex_ingester_tsdb_head_out_of_order_samples_appended_total counter + cortex_ingester_tsdb_head_out_of_order_samples_appended_total{type="float",user="test"} 0 + # HELP cortex_ingester_tsdb_head_samples_appended_total Total number of appended samples. + # TYPE cortex_ingester_tsdb_head_samples_appended_total counter + cortex_ingester_tsdb_head_samples_appended_total{type="float",user="test"} 0 + cortex_ingester_tsdb_head_samples_appended_total{type="histogram",user="test"} 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "should fail to ingest histogram due to OOO native histogram. Sample and histogram has same timestamp but sample got ingested first": { + reqs: []*cortexpbv2.WriteRequest{ + cortexpbv2.ToWriteRequestV2( + []labels.Labels{metricLabels}, + []cortexpbv2.Sample{{Value: 2, Timestamp: 10}}, + []cortexpbv2.Histogram{testHistogramV2}, + nil, + cortexpbv2.API), + }, + expectedErr: nil, + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 10}}}, + }, + additionalMetrics: []string{ + "cortex_ingester_tsdb_head_samples_appended_total", + "cortex_ingester_tsdb_out_of_order_samples_total", + "cortex_ingester_active_series", + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 2 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_tsdb_head_samples_appended_total Total number of appended samples. + # TYPE cortex_ingester_tsdb_head_samples_appended_total counter + cortex_ingester_tsdb_head_samples_appended_total{type="float",user="test"} 1 + cortex_ingester_tsdb_head_samples_appended_total{type="histogram",user="test"} 0 + # HELP cortex_ingester_tsdb_out_of_order_samples_total Total number of out of order samples ingestion failed attempts due to out of order being disabled. + # TYPE cortex_ingester_tsdb_out_of_order_samples_total counter + cortex_ingester_tsdb_out_of_order_samples_total{type="float",user="test"} 0 + cortex_ingester_tsdb_out_of_order_samples_total{type="histogram",user="test"} 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + registry := prometheus.NewRegistry() + + // Create a mocked ingester + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries + + limits := defaultLimitsTestConfig() + limits.MaxExemplars = testData.maxExemplars + limits.OutOfOrderTimeWindow = model.Duration(testData.oooTimeWindow) + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry, !testData.disableNativeHistogram) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + ctx := user.InjectOrgID(context.Background(), userID) + + // Wait until the ingester is ACTIVE + test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push timeseries + for idx, req := range testData.reqs { + _, err := i.PushV2(ctx, req) + + // We expect no error on any request except the last one + // which may error (and in that case we assert on it) + if idx < len(testData.reqs)-1 { + assert.NoError(t, err) + } else { + assert.Equal(t, testData.expectedErr, err) + } + } + + // Read back samples to see what has been really ingested + s := &mockQueryStreamServer{ctx: ctx} + err = i.QueryStream(&client.QueryRequest{ + StartTimestampMs: math.MinInt64, + EndTimestampMs: math.MaxInt64, + Matchers: []*client.LabelMatcher{{Type: client.REGEX_MATCH, Name: labels.MetricName, Value: ".*"}}, + }, s) + require.NoError(t, err) + set, err := seriesSetFromResponseStream(s) + require.NoError(t, err) + + require.NotNil(t, set) + r, err := client.SeriesSetToQueryResponse(set) + require.NoError(t, err) + assert.Equal(t, testData.expectedIngested, r.Timeseries) + + // Read back samples to see what has been really ingested + exemplarRes, err := i.QueryExemplars(ctx, &client.ExemplarQueryRequest{ + StartTimestampMs: math.MinInt64, + EndTimestampMs: math.MaxInt64, + Matchers: []*client.LabelMatchers{ + {Matchers: []*client.LabelMatcher{{Type: client.REGEX_MATCH, Name: labels.MetricName, Value: ".*"}}}, + }, + }) + + require.NoError(t, err) + require.NotNil(t, exemplarRes) + assert.Equal(t, testData.expectedExemplarsIngested, exemplarRes.Timeseries) + + // Read back metadata to see what has been really ingested. + mres, err := i.MetricsMetadata(ctx, &client.MetricsMetadataRequest{}) + + require.NoError(t, err) + require.NotNil(t, mres) + + // Order is never guaranteed. + assert.ElementsMatch(t, testData.expectedMetadataIngested, mres.Metadata) + + // Update active series for metrics check. + if !testData.disableActiveSeries { + i.updateActiveSeries(ctx) + } + + // Append additional metrics to assert on. + mn := append(metricNames, testData.additionalMetrics...) + + // Check tracked Prometheus metrics + err = testutil.GatherAndCompare(registry, strings.NewReader(testData.expectedMetrics), mn...) + assert.NoError(t, err) + }) + } +} + +func TestIngesterPRW2_MetricLimitExceeded(t *testing.T) { + limits := defaultLimitsTestConfig() + limits.MaxLocalSeriesPerMetric = 1 + limits.MaxLocalMetadataPerMetric = 1 + + dir := t.TempDir() + + chunksDir := filepath.Join(dir, "chunks") + blocksDir := filepath.Join(dir, "blocks") + require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) + require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) + + blocksIngesterGenerator := func() *Ingester { + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, prometheus.NewRegistry(), true) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) + // Wait until it's ACTIVE + test.Poll(t, time.Second, ring.ACTIVE, func() interface{} { + return ing.lifecycler.GetState() + }) + + return ing + } + + tests := []string{"chunks", "blocks"} + for i, ingGenerator := range []func() *Ingester{blocksIngesterGenerator} { + t.Run(tests[i], func(t *testing.T) { + ing := ingGenerator() + + userID := "1" + labels1 := labels.Labels{{Name: labels.MetricName, Value: "testmetric"}, {Name: "foo", Value: "bar"}} + sample1 := cortexpbv2.Sample{ + Timestamp: 0, + Value: 1, + } + sample2 := cortexpbv2.Sample{ + Timestamp: 1, + Value: 2, + } + labels3 := labels.Labels{{Name: labels.MetricName, Value: "testmetric"}, {Name: "foo", Value: "biz"}} + sample3 := cortexpbv2.Sample{ + Timestamp: 1, + Value: 3, + } + + // Append only one series and one metadata first, expect no error. + ctx := user.InjectOrgID(context.Background(), userID) + _, err := ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2([]labels.Labels{labels1}, []cortexpbv2.Sample{sample1}, nil, []cortexpbv2.Metadata{{HelpRef: 5, Type: cortexpbv2.METRIC_TYPE_COUNTER}}, cortexpbv2.API, "a help for testmetric")) + require.NoError(t, err) + + testLimits := func() { + // Append two series, expect series-exceeded error. + _, err = ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2([]labels.Labels{labels1, labels3}, []cortexpbv2.Sample{sample2, sample3}, nil, nil, cortexpbv2.API)) + httpResp, ok := httpgrpc.HTTPResponseFromError(err) + require.True(t, ok, "returned error is not an httpgrpc response") + assert.Equal(t, http.StatusBadRequest, int(httpResp.Code)) + assert.Equal(t, wrapWithUser(makeMetricLimitError(perMetricSeriesLimit, labels3, ing.limiter.FormatError(userID, errMaxSeriesPerMetricLimitExceeded)), userID).Error(), string(httpResp.Body)) + + // Append two metadata for the same metric. Drop the second one, and expect no error since metadata is a best effort approach. + _, err = ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2([]labels.Labels{labels1, labels3}, nil, nil, []cortexpbv2.Metadata{{HelpRef: 6, Type: cortexpbv2.METRIC_TYPE_COUNTER}, {HelpRef: 7, Type: cortexpbv2.METRIC_TYPE_COUNTER}}, cortexpbv2.API, "a help for testmetric", "a help for testmetric2")) + require.NoError(t, err) + + // Read samples back via ingester queries. + res, _, err := runTestQuery(ctx, t, ing, labels.MatchEqual, model.MetricNameLabel, "testmetric") + require.NoError(t, err) + + // Verify Series + expected := model.Matrix{ + { + Metric: cortexpb.FromLabelAdaptersToMetric(cortexpb.FromLabelsToLabelAdapters(labels1)), + Values: []model.SamplePair{ + { + Timestamp: model.Time(sample1.Timestamp), + Value: model.SampleValue(sample1.Value), + }, + { + Timestamp: model.Time(sample2.Timestamp), + Value: model.SampleValue(sample2.Value), + }, + }, + }, + } + + assert.Equal(t, expected, res) + + // Verify metadata + m, err := ing.MetricsMetadata(ctx, nil) + require.NoError(t, err) + resultMetadata := &cortexpb.MetricMetadata{MetricFamilyName: "testmetric", Help: "a help for testmetric", Type: cortexpb.COUNTER} + assert.Equal(t, []*cortexpb.MetricMetadata{resultMetadata}, m.Metadata) + } + + testLimits() + + // Limits should hold after restart. + services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck + ing = ingGenerator() + defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck + + testLimits() + }) + } +} + +func TestIngesterPRW2_UserLimitExceeded(t *testing.T) { + limits := defaultLimitsTestConfig() + limits.MaxLocalSeriesPerUser = 1 + limits.MaxLocalMetricsWithMetadataPerUser = 1 + + dir := t.TempDir() + + chunksDir := filepath.Join(dir, "chunks") + blocksDir := filepath.Join(dir, "blocks") + require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) + require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) + + blocksIngesterGenerator := func() *Ingester { + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, prometheus.NewRegistry(), true) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) + // Wait until it's ACTIVE + test.Poll(t, time.Second, ring.ACTIVE, func() interface{} { + return ing.lifecycler.GetState() + }) + + return ing + } + + tests := []string{"blocks"} + for i, ingGenerator := range []func() *Ingester{blocksIngesterGenerator} { + t.Run(tests[i], func(t *testing.T) { + ing := ingGenerator() + + userID := "1" + // Series + labels1 := labels.Labels{{Name: labels.MetricName, Value: "testmetric"}, {Name: "foo", Value: "bar"}} + sample1 := cortexpbv2.Sample{ + Timestamp: 0, + Value: 1, + } + sample2 := cortexpbv2.Sample{ + Timestamp: 1, + Value: 2, + } + labels3 := labels.Labels{{Name: labels.MetricName, Value: "testmetric2"}, {Name: "foo", Value: "biz"}} + sample3 := cortexpbv2.Sample{ + Timestamp: 1, + Value: 3, + } + + // Append only one series and one metadata first, expect no error. + ctx := user.InjectOrgID(context.Background(), userID) + _, err := ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2([]labels.Labels{labels1}, []cortexpbv2.Sample{sample1}, nil, []cortexpbv2.Metadata{{HelpRef: 5, Type: cortexpbv2.METRIC_TYPE_COUNTER}}, cortexpbv2.API, "a help for testmetric")) + require.NoError(t, err) + + testLimits := func() { + // Append to two series, expect series-exceeded error. + _, err = ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2([]labels.Labels{labels1, labels3}, []cortexpbv2.Sample{sample2, sample3}, nil, nil, cortexpbv2.API)) + httpResp, ok := httpgrpc.HTTPResponseFromError(err) + require.True(t, ok, "returned error is not an httpgrpc response") + assert.Equal(t, http.StatusBadRequest, int(httpResp.Code)) + assert.Equal(t, wrapWithUser(makeLimitError(perUserSeriesLimit, ing.limiter.FormatError(userID, errMaxSeriesPerUserLimitExceeded)), userID).Error(), string(httpResp.Body)) + + // Append two metadata, expect no error since metadata is a best effort approach. + _, err = ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2([]labels.Labels{labels1, labels3}, nil, nil, []cortexpbv2.Metadata{{HelpRef: 7, Type: cortexpbv2.METRIC_TYPE_COUNTER}, {HelpRef: 8, Type: cortexpbv2.METRIC_TYPE_COUNTER}}, cortexpbv2.API, "a help for testmetric", "a help for testmetric2")) + require.NoError(t, err) + + // Read samples back via ingester queries. + res, _, err := runTestQuery(ctx, t, ing, labels.MatchEqual, model.MetricNameLabel, "testmetric") + require.NoError(t, err) + + expected := model.Matrix{ + { + Metric: cortexpb.FromLabelAdaptersToMetric(cortexpb.FromLabelsToLabelAdapters(labels1)), + Values: []model.SamplePair{ + { + Timestamp: model.Time(sample1.Timestamp), + Value: model.SampleValue(sample1.Value), + }, + { + Timestamp: model.Time(sample2.Timestamp), + Value: model.SampleValue(sample2.Value), + }, + }, + }, + } + + // Verify samples + require.Equal(t, expected, res) + + // Verify metadata + m, err := ing.MetricsMetadata(ctx, nil) + require.NoError(t, err) + resultMetadata := &cortexpb.MetricMetadata{MetricFamilyName: "testmetric", Help: "a help for testmetric", Type: cortexpb.COUNTER} + assert.Equal(t, []*cortexpb.MetricMetadata{resultMetadata}, m.Metadata) + } + + testLimits() + + // Limits should hold after restart. + services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck + ing = ingGenerator() + defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck + + testLimits() + }) + } + +} + +func TestIngesterPRW2_PerLabelsetLimitExceeded(t *testing.T) { + limits := defaultLimitsTestConfig() + userID := "1" + registry := prometheus.NewRegistry() + + limits.LimitsPerLabelSet = []validation.LimitsPerLabelSet{ + { + LabelSet: labels.FromMap(map[string]string{ + "label1": "value1", + }), + Limits: validation.LimitsPerLabelSetEntry{ + MaxSeries: 3, + }, + }, + { + LabelSet: labels.FromMap(map[string]string{ + "label2": "value2", + }), + Limits: validation.LimitsPerLabelSetEntry{ + MaxSeries: 2, + }, + }, + } + tenantLimits := newMockTenantLimits(map[string]*validation.Limits{userID: &limits}) + + b, err := json.Marshal(limits) + require.NoError(t, err) + require.NoError(t, limits.UnmarshalJSON(b)) + + dir := t.TempDir() + chunksDir := filepath.Join(dir, "chunks") + blocksDir := filepath.Join(dir, "blocks") + require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) + require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) + + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, tenantLimits, blocksDir, registry, true) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) + // Wait until it's ACTIVE + test.Poll(t, time.Second, ring.ACTIVE, func() interface{} { + return ing.lifecycler.GetState() + }) + + ctx := user.InjectOrgID(context.Background(), userID) + samples := []cortexpbv2.Sample{{Value: 2, Timestamp: 10}} + + // Create first series within the limits + for _, set := range limits.LimitsPerLabelSet { + lbls := []string{labels.MetricName, "metric_name"} + for _, lbl := range set.LabelSet { + lbls = append(lbls, lbl.Name, lbl.Value) + } + for i := 0; i < set.Limits.MaxSeries; i++ { + _, err = ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2( + []labels.Labels{labels.FromStrings(append(lbls, "extraLabel", fmt.Sprintf("extraValue%v", i))...)}, samples, nil, nil, cortexpbv2.API)) + require.NoError(t, err) + } + } + + ing.updateActiveSeries(ctx) + require.NoError(t, testutil.GatherAndCompare(registry, bytes.NewBufferString(` + # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. + # TYPE cortex_ingester_limits_per_labelset gauge + cortex_ingester_limits_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_limits_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + # HELP cortex_ingester_usage_per_labelset Current usage per user and labelset. + # TYPE cortex_ingester_usage_per_labelset gauge + cortex_ingester_usage_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_usage_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + `), "cortex_ingester_usage_per_labelset", "cortex_ingester_limits_per_labelset")) + + // Should impose limits + for _, set := range limits.LimitsPerLabelSet { + lbls := []string{labels.MetricName, "metric_name"} + for _, lbl := range set.LabelSet { + lbls = append(lbls, lbl.Name, lbl.Value) + } + _, err = ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2( + []labels.Labels{labels.FromStrings(append(lbls, "newLabel", "newValue")...)}, samples, nil, nil, cortexpbv2.API)) + httpResp, ok := httpgrpc.HTTPResponseFromError(err) + require.True(t, ok, "returned error is not an httpgrpc response") + assert.Equal(t, http.StatusBadRequest, int(httpResp.Code)) + require.ErrorContains(t, err, set.Id) + } + + ing.updateActiveSeries(ctx) + require.NoError(t, testutil.GatherAndCompare(registry, bytes.NewBufferString(` + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 2 + # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. + # TYPE cortex_ingester_limits_per_labelset gauge + cortex_ingester_limits_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_limits_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + # HELP cortex_ingester_usage_per_labelset Current usage per user and labelset. + # TYPE cortex_ingester_usage_per_labelset gauge + cortex_ingester_usage_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_usage_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + `), "cortex_ingester_usage_per_labelset", "cortex_ingester_limits_per_labelset", "cortex_discarded_samples_total")) + + // Should apply composite limits + limits.LimitsPerLabelSet = append(limits.LimitsPerLabelSet, + validation.LimitsPerLabelSet{LabelSet: labels.FromMap(map[string]string{ + "comp1": "compValue1", + }), + Limits: validation.LimitsPerLabelSetEntry{ + MaxSeries: 10, + }, + }, + validation.LimitsPerLabelSet{LabelSet: labels.FromMap(map[string]string{ + "comp2": "compValue2", + }), + Limits: validation.LimitsPerLabelSetEntry{ + MaxSeries: 10, + }, + }, + validation.LimitsPerLabelSet{LabelSet: labels.FromMap(map[string]string{ + "comp1": "compValue1", + "comp2": "compValue2", + }), + Limits: validation.LimitsPerLabelSetEntry{ + MaxSeries: 2, + }, + }, + ) + + b, err = json.Marshal(limits) + require.NoError(t, err) + require.NoError(t, limits.UnmarshalJSON(b)) + tenantLimits.setLimits(userID, &limits) + + // Should backfill + ing.updateActiveSeries(ctx) + require.NoError(t, testutil.GatherAndCompare(registry, bytes.NewBufferString(` + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 2 + # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. + # TYPE cortex_ingester_limits_per_labelset gauge + cortex_ingester_limits_per_labelset{labelset="{comp1=\"compValue1\", comp2=\"compValue2\"}",limit="max_series",user="1"} 2 + cortex_ingester_limits_per_labelset{labelset="{comp1=\"compValue1\"}",limit="max_series",user="1"} 10 + cortex_ingester_limits_per_labelset{labelset="{comp2=\"compValue2\"}",limit="max_series",user="1"} 10 + cortex_ingester_limits_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_limits_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + # HELP cortex_ingester_usage_per_labelset Current usage per user and labelset. + # TYPE cortex_ingester_usage_per_labelset gauge + cortex_ingester_usage_per_labelset{labelset="{comp1=\"compValue1\", comp2=\"compValue2\"}",limit="max_series",user="1"} 0 + cortex_ingester_usage_per_labelset{labelset="{comp1=\"compValue1\"}",limit="max_series",user="1"} 0 + cortex_ingester_usage_per_labelset{labelset="{comp2=\"compValue2\"}",limit="max_series",user="1"} 0 + cortex_ingester_usage_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_usage_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + `), "cortex_ingester_usage_per_labelset", "cortex_ingester_limits_per_labelset", "cortex_discarded_samples_total")) + + // Adding 5 metrics with only 1 label + for i := 0; i < 5; i++ { + lbls := []string{labels.MetricName, "metric_name", "comp1", "compValue1"} + _, err = ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2( + []labels.Labels{labels.FromStrings(append(lbls, "extraLabel", fmt.Sprintf("extraValue%v", i))...)}, samples, nil, nil, cortexpbv2.API)) + require.NoError(t, err) + } + + // Adding 2 metrics with both labels (still below the limit) + lbls := []string{labels.MetricName, "metric_name", "comp1", "compValue1", "comp2", "compValue2"} + for i := 0; i < 2; i++ { + _, err = ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2( + []labels.Labels{labels.FromStrings(append(lbls, "extraLabel", fmt.Sprintf("extraValue%v", i))...)}, samples, nil, nil, cortexpbv2.API)) + require.NoError(t, err) + } + + // Now we should hit the limit as we already have 2 metrics with comp1=compValue1, comp2=compValue2 + _, err = ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2( + []labels.Labels{labels.FromStrings(append(lbls, "newLabel", "newValue")...)}, samples, nil, nil, cortexpbv2.API)) + httpResp, ok := httpgrpc.HTTPResponseFromError(err) + require.True(t, ok, "returned error is not an httpgrpc response") + assert.Equal(t, http.StatusBadRequest, int(httpResp.Code)) + require.ErrorContains(t, err, labels.FromStrings("comp1", "compValue1", "comp2", "compValue2").String()) + + ing.updateActiveSeries(ctx) + require.NoError(t, testutil.GatherAndCompare(registry, bytes.NewBufferString(` + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 3 + # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. + # TYPE cortex_ingester_limits_per_labelset gauge + cortex_ingester_limits_per_labelset{labelset="{comp1=\"compValue1\", comp2=\"compValue2\"}",limit="max_series",user="1"} 2 + cortex_ingester_limits_per_labelset{labelset="{comp1=\"compValue1\"}",limit="max_series",user="1"} 10 + cortex_ingester_limits_per_labelset{labelset="{comp2=\"compValue2\"}",limit="max_series",user="1"} 10 + cortex_ingester_limits_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_limits_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + # HELP cortex_ingester_usage_per_labelset Current usage per user and labelset. + # TYPE cortex_ingester_usage_per_labelset gauge + cortex_ingester_usage_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_usage_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + cortex_ingester_usage_per_labelset{labelset="{comp1=\"compValue1\", comp2=\"compValue2\"}",limit="max_series",user="1"} 2 + cortex_ingester_usage_per_labelset{labelset="{comp1=\"compValue1\"}",limit="max_series",user="1"} 7 + cortex_ingester_usage_per_labelset{labelset="{comp2=\"compValue2\"}",limit="max_series",user="1"} 2 + `), "cortex_ingester_usage_per_labelset", "cortex_ingester_limits_per_labelset", "cortex_discarded_samples_total")) + + // Should bootstrap and apply limits when configuration change + limits.LimitsPerLabelSet = append(limits.LimitsPerLabelSet, + validation.LimitsPerLabelSet{LabelSet: labels.FromMap(map[string]string{ + labels.MetricName: "metric_name", + "comp2": "compValue2", + }), + Limits: validation.LimitsPerLabelSetEntry{ + MaxSeries: 3, // we already have 2 so we need to allow 1 more + }, + }, + ) + + b, err = json.Marshal(limits) + require.NoError(t, err) + require.NoError(t, limits.UnmarshalJSON(b)) + tenantLimits.setLimits(userID, &limits) + + lbls = []string{labels.MetricName, "metric_name", "comp2", "compValue2"} + _, err = ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2( + []labels.Labels{labels.FromStrings(append(lbls, "extraLabel", "extraValueUpdate")...)}, samples, nil, nil, cortexpbv2.API)) + require.NoError(t, err) + + _, err = ing.PushV2(ctx, cortexpbv2.ToWriteRequestV2( + []labels.Labels{labels.FromStrings(append(lbls, "extraLabel", "extraValueUpdate2")...)}, samples, nil, nil, cortexpbv2.API)) + httpResp, ok = httpgrpc.HTTPResponseFromError(err) + require.True(t, ok, "returned error is not an httpgrpc response") + assert.Equal(t, http.StatusBadRequest, int(httpResp.Code)) + require.ErrorContains(t, err, labels.FromStrings(lbls...).String()) + + ing.updateActiveSeries(ctx) + require.NoError(t, testutil.GatherAndCompare(registry, bytes.NewBufferString(` + # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. + # TYPE cortex_ingester_limits_per_labelset gauge + cortex_ingester_limits_per_labelset{labelset="{__name__=\"metric_name\", comp2=\"compValue2\"}",limit="max_series",user="1"} 3 + cortex_ingester_limits_per_labelset{labelset="{comp1=\"compValue1\", comp2=\"compValue2\"}",limit="max_series",user="1"} 2 + cortex_ingester_limits_per_labelset{labelset="{comp1=\"compValue1\"}",limit="max_series",user="1"} 10 + cortex_ingester_limits_per_labelset{labelset="{comp2=\"compValue2\"}",limit="max_series",user="1"} 10 + cortex_ingester_limits_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_limits_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + # HELP cortex_ingester_usage_per_labelset Current usage per user and labelset. + # TYPE cortex_ingester_usage_per_labelset gauge + cortex_ingester_usage_per_labelset{labelset="{__name__=\"metric_name\", comp2=\"compValue2\"}",limit="max_series",user="1"} 3 + cortex_ingester_usage_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_usage_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + cortex_ingester_usage_per_labelset{labelset="{comp1=\"compValue1\", comp2=\"compValue2\"}",limit="max_series",user="1"} 2 + cortex_ingester_usage_per_labelset{labelset="{comp1=\"compValue1\"}",limit="max_series",user="1"} 7 + cortex_ingester_usage_per_labelset{labelset="{comp2=\"compValue2\"}",limit="max_series",user="1"} 3 + `), "cortex_ingester_usage_per_labelset", "cortex_ingester_limits_per_labelset")) + + // Should remove metrics when the limits is removed + limits.LimitsPerLabelSet = limits.LimitsPerLabelSet[:2] + b, err = json.Marshal(limits) + require.NoError(t, err) + require.NoError(t, limits.UnmarshalJSON(b)) + tenantLimits.setLimits(userID, &limits) + ing.updateActiveSeries(ctx) + require.NoError(t, testutil.GatherAndCompare(registry, bytes.NewBufferString(` + # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. + # TYPE cortex_ingester_limits_per_labelset gauge + cortex_ingester_limits_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_limits_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + # HELP cortex_ingester_usage_per_labelset Current usage per user and labelset. + # TYPE cortex_ingester_usage_per_labelset gauge + cortex_ingester_usage_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_usage_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + `), "cortex_ingester_usage_per_labelset", "cortex_ingester_limits_per_labelset")) + + // Should persist between restarts + services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck + registry = prometheus.NewRegistry() + ing, err = prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, tenantLimits, blocksDir, registry, true) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) + ing.updateActiveSeries(ctx) + require.NoError(t, testutil.GatherAndCompare(registry, bytes.NewBufferString(` + # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. + # TYPE cortex_ingester_limits_per_labelset gauge + cortex_ingester_limits_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_limits_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + # HELP cortex_ingester_usage_per_labelset Current usage per user and labelset. + # TYPE cortex_ingester_usage_per_labelset gauge + cortex_ingester_usage_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 + cortex_ingester_usage_per_labelset{labelset="{label2=\"value2\"}",limit="max_series",user="1"} 2 + `), "cortex_ingester_usage_per_labelset", "cortex_ingester_limits_per_labelset")) + services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck + +} // Referred from https://github.com/prometheus/prometheus/blob/v2.52.1/model/histogram/histogram_test.go#L985. func TestIngesterPRW2_PushNativeHistogramErrors(t *testing.T) { @@ -218,7 +1577,7 @@ func TestIngesterPRW2_PushNativeHistogramErrors(t *testing.T) { return i.lifecycler.GetState() }) - req := cortexpbv2.ToWriteRequestV2([]labels.Labels{metricLabels}, []string{"", "__name__", "test"}, nil, tc.histograms, nil, cortexpbv2.API) + req := cortexpbv2.ToWriteRequestV2([]labels.Labels{metricLabels}, nil, tc.histograms, nil, cortexpbv2.API) // Push timeseries _, err = i.PushV2(ctx, req) assert.Equal(t, httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestErr(tc.expectedErr, model.Time(10), metricLabelAdapters), userID).Error()), err) @@ -231,7 +1590,6 @@ func TestIngesterPRW2_PushNativeHistogramErrors(t *testing.T) { func TestIngesterPRW2_Push_ShouldCorrectlyTrackMetricsInMultiTenantScenario(t *testing.T) { metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) - symbols := []string{"", "__name__", "test"} metricNames := []string{ "cortex_ingester_ingested_samples_total", "cortex_ingester_ingested_samples_failures_total", @@ -263,14 +1621,12 @@ func TestIngesterPRW2_Push_ShouldCorrectlyTrackMetricsInMultiTenantScenario(t *t reqs := []*cortexpbv2.WriteRequest{ cortexpbv2.ToWriteRequestV2( []labels.Labels{metricLabels}, - symbols, []cortexpbv2.Sample{{Value: 1, Timestamp: 9}}, nil, nil, cortexpbv2.API), cortexpbv2.ToWriteRequestV2( []labels.Labels{metricLabels}, - symbols, []cortexpbv2.Sample{{Value: 2, Timestamp: 10}}, nil, nil, @@ -321,7 +1677,6 @@ func TestIngesterPRW2_Push_ShouldCorrectlyTrackMetricsInMultiTenantScenario(t *t func TestIngesterPRW2_Push_DecreaseInactiveSeries(t *testing.T) { metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) - symbols := []string{"", "__name__", "test"} metricNames := []string{ "cortex_ingester_memory_series_created_total", "cortex_ingester_memory_series_removed_total", @@ -350,14 +1705,12 @@ func TestIngesterPRW2_Push_DecreaseInactiveSeries(t *testing.T) { reqs := []*cortexpbv2.WriteRequest{ cortexpbv2.ToWriteRequestV2( []labels.Labels{metricLabels}, - symbols, []cortexpbv2.Sample{{Value: 1, Timestamp: 9}}, nil, nil, cortexpbv2.API), cortexpbv2.ToWriteRequestV2( []labels.Labels{metricLabels}, - symbols, []cortexpbv2.Sample{{Value: 2, Timestamp: 10}}, nil, nil, @@ -424,15 +1777,8 @@ func benchmarkIngesterPRW2Push(b *testing.B, limits validation.Limits, errorsExp metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) startTime := util.TimeToMillis(time.Now()) - st := writev2.NewSymbolTable() - for _, lb := range metricLabels { - st.Symbolize(lb.Name) - st.Symbolize(lb.Value) - } - currTimeReq := cortexpbv2.ToWriteRequestV2( []labels.Labels{metricLabels}, - st.Symbols(), []cortexpbv2.Sample{{Value: 1, Timestamp: startTime}}, nil, nil, @@ -446,13 +1792,6 @@ func benchmarkIngesterPRW2Push(b *testing.B, limits validation.Limits, errorsExp ) allLabels, allSamples := benchmarkDataV2(series) - st2 := writev2.NewSymbolTable() - for _, lbs := range allLabels { - for _, lb := range lbs { - st2.Symbolize(lb.Name) - st2.Symbolize(lb.Value) - } - } b.ResetTimer() for iter := 0; iter < b.N; iter++ { @@ -461,7 +1800,7 @@ func benchmarkIngesterPRW2Push(b *testing.B, limits validation.Limits, errorsExp for i := range allSamples { allSamples[i].Timestamp = startTime + int64(iter*samples+j+1) } - _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(allLabels, st2.Symbols(), allSamples, nil, nil, cortexpbv2.API)) + _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(allLabels, allSamples, nil, nil, cortexpbv2.API)) if !errorsExpected { require.NoError(b, err) } @@ -511,7 +1850,6 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { // Push a single time series to set the TSDB min time. currTimeReq := cortexpbv2.ToWriteRequestV2( []labels.Labels{{{Name: labels.MetricName, Value: metricName}}}, - []string{"", "__name__", metricName}, []cortexpbv2.Sample{{Value: 1, Timestamp: util.TimeToMillis(time.Now())}}, nil, nil, @@ -521,16 +1859,9 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { }, runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpbv2.Sample) { expectedErr := storage.ErrOutOfBounds.Error() - st := writev2.NewSymbolTable() - for _, lbs := range metrics { - for _, lb := range lbs { - st.Symbolize(lb.Name) - st.Symbolize(lb.Value) - } - } // Push out of bound samples. for n := 0; n < b.N; n++ { - _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, st.Symbols(), samples, nil, nil, cortexpbv2.API)) // nolint:errcheck + _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, samples, nil, nil, cortexpbv2.API)) // nolint:errcheck verifyErrorString(b, err, expectedErr) } @@ -543,7 +1874,6 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { for i := 0; i < numSeriesPerRequest; i++ { currTimeReq := cortexpbv2.ToWriteRequestV2( []labels.Labels{{{Name: labels.MetricName, Value: metricName}, {Name: "cardinality", Value: strconv.Itoa(i)}}}, - []string{"", "__name__", metricName, "cardinality", strconv.Itoa(i)}, []cortexpbv2.Sample{{Value: 1, Timestamp: sampleTimestamp + 1}}, nil, nil, @@ -565,7 +1895,7 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { } // Push out of order samples. for n := 0; n < b.N; n++ { - _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, st.Symbols(), samples, nil, nil, cortexpbv2.API)) // nolint:errcheck + _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, samples, nil, nil, cortexpbv2.API)) // nolint:errcheck verifyErrorString(b, err, expectedErr) } @@ -580,7 +1910,6 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { // Push a series with a metric name different than the one used during the benchmark. currTimeReq := cortexpbv2.ToWriteRequestV2( []labels.Labels{labels.FromStrings(labels.MetricName, "another")}, - []string{"", "__name__", "another"}, []cortexpbv2.Sample{{Value: 1, Timestamp: sampleTimestamp + 1}}, nil, nil, @@ -599,7 +1928,7 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { } } for n := 0; n < b.N; n++ { - _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, st.Symbols(), samples, nil, nil, cortexpbv2.API)) // nolint:errcheck + _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, samples, nil, nil, cortexpbv2.API)) // nolint:errcheck verifyErrorString(b, err, "per-user series limit") } }, @@ -613,7 +1942,6 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { // Push a series with the same metric name but different labels than the one used during the benchmark. currTimeReq := cortexpbv2.ToWriteRequestV2( []labels.Labels{labels.FromStrings(labels.MetricName, metricName, "cardinality", "another")}, - []string{"", "__name__", metricName, "cardinality", "another"}, []cortexpbv2.Sample{{Value: 1, Timestamp: sampleTimestamp + 1}}, nil, nil, @@ -631,7 +1959,7 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { } // Push series with different labels than the one already pushed. for n := 0; n < b.N; n++ { - _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, st.Symbols(), samples, nil, nil, cortexpbv2.API)) // nolint:errcheck + _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, samples, nil, nil, cortexpbv2.API)) // nolint:errcheck verifyErrorString(b, err, "per-metric series limit") } }, @@ -661,7 +1989,7 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { } // Push series with different labels than the one already pushed. for n := 0; n < b.N; n++ { - _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, st.Symbols(), samples, nil, nil, cortexpbv2.API)) + _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, samples, nil, nil, cortexpbv2.API)) verifyErrorString(b, err, "push rate reached") } }, @@ -690,7 +2018,7 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { } // Push series with different labels than the one already pushed. for n := 0; n < b.N; n++ { - _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, st.Symbols(), samples, nil, nil, cortexpbv2.API)) + _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, samples, nil, nil, cortexpbv2.API)) verifyErrorString(b, err, "max tenants limit reached") } }, @@ -716,7 +2044,7 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { } } for n := 0; n < b.N; n++ { - _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, st.Symbols(), samples, nil, nil, cortexpbv2.API)) + _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, samples, nil, nil, cortexpbv2.API)) verifyErrorString(b, err, "max series limit reached") } }, @@ -741,7 +2069,7 @@ func Benchmark_IngesterPRW2_PushOnError(b *testing.B) { } } for n := 0; n < b.N; n++ { - _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, st.Symbols(), samples, nil, nil, cortexpbv2.API)) + _, err := ingester.PushV2(ctx, cortexpbv2.ToWriteRequestV2(metrics, samples, nil, nil, cortexpbv2.API)) verifyErrorString(b, err, "too many inflight push requests") } }, @@ -2405,9 +3733,7 @@ func TestIngesterPRW2_CompactAndCloseIdleTSDB(t *testing.T) { require.Equal(t, int64(0), i.TSDBState.seriesCount.Load()) // Flushing removed all series from memory. // Verify that user has disappeared from metrics. - err = testutil.GatherAndCompare(r, strings.NewReader(""), userMetrics...) - require.ErrorContains(t, err, "expected metric name(s) not found") - require.ErrorContains(t, err, strings.Join(userMetrics, " ")) + require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(""), userMetrics...)) require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` # HELP cortex_ingester_memory_users The current number of users in memory. @@ -2696,13 +4022,13 @@ func TestIngesterPRW2_PushInstanceLimits(t *testing.T) { "test": { cortexpbv2.ToWriteRequestV2( []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}})}, - []string{"", "__name__", "test", "a help for metric_name_1"}, []cortexpbv2.Sample{{Value: 1, Timestamp: 9}}, nil, []cortexpbv2.Metadata{ {Type: cortexpbv2.METRIC_TYPE_COUNTER, HelpRef: 3}, }, - cortexpbv2.API), + cortexpbv2.API, + "a help for metric_name_1"), }, }, expectedErr: nil, @@ -2714,7 +4040,6 @@ func TestIngesterPRW2_PushInstanceLimits(t *testing.T) { "test": { cortexpbv2.ToWriteRequestV2( []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, - []string{"", "__name__", "test1"}, []cortexpbv2.Sample{{Value: 1, Timestamp: 9}}, nil, nil, @@ -2722,7 +4047,6 @@ func TestIngesterPRW2_PushInstanceLimits(t *testing.T) { cortexpbv2.ToWriteRequestV2( []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test2"}})}, // another series - []string{"", "__name__", "test2"}, []cortexpbv2.Sample{{Value: 1, Timestamp: 10}}, nil, nil, @@ -2739,7 +4063,6 @@ func TestIngesterPRW2_PushInstanceLimits(t *testing.T) { "user1": { cortexpbv2.ToWriteRequestV2( []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, - []string{"", "__name__", "test1"}, []cortexpbv2.Sample{{Value: 1, Timestamp: 9}}, nil, nil, @@ -2749,7 +4072,6 @@ func TestIngesterPRW2_PushInstanceLimits(t *testing.T) { "user2": { cortexpbv2.ToWriteRequestV2( []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test2"}})}, // another series - []string{"", "__name__", "test2"}, []cortexpbv2.Sample{{Value: 1, Timestamp: 10}}, nil, nil, @@ -2765,7 +4087,6 @@ func TestIngesterPRW2_PushInstanceLimits(t *testing.T) { "user1": { cortexpbv2.ToWriteRequestV2( []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, - []string{"", "__name__", "test1"}, []cortexpbv2.Sample{{Value: 1, Timestamp: 9}}, nil, nil, @@ -2773,7 +4094,6 @@ func TestIngesterPRW2_PushInstanceLimits(t *testing.T) { cortexpbv2.ToWriteRequestV2( []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, - []string{"", "__name__", "test1"}, []cortexpbv2.Sample{{Value: 1, Timestamp: 10}}, nil, nil, @@ -2927,12 +4247,6 @@ func TestIngesterPRW2_QueryExemplar_MaxInflightQueryRequest(t *testing.T) { } func generateSamplesForLabelV2(lbs labels.Labels, count int) *cortexpbv2.WriteRequest { - st := writev2.NewSymbolTable() - for _, lb := range lbs { - st.Symbolize(lb.Name) - st.Symbolize(lb.Value) - } - var lbls = make([]labels.Labels, 0, count) var samples = make([]cortexpbv2.Sample, 0, count) @@ -2944,10 +4258,10 @@ func generateSamplesForLabelV2(lbs labels.Labels, count int) *cortexpbv2.WriteRe lbls = append(lbls, lbs) } - return cortexpbv2.ToWriteRequestV2(lbls, st.Symbols(), samples, nil, nil, cortexpbv2.API) + return cortexpbv2.ToWriteRequestV2(lbls, samples, nil, nil, cortexpbv2.API) } -func mockWriteRequestWithMetadataV2(t *testing.T, lbls labels.Labels, symbols []string, value float64, timestamp int64, metadata cortexpbv2.Metadata) (*cortexpbv2.WriteRequest, *client.QueryStreamResponse) { +func mockWriteRequestWithMetadataV2(t *testing.T, lbls labels.Labels, value float64, timestamp int64, metadata cortexpbv2.Metadata, additionalSymbols ...string) (*cortexpbv2.WriteRequest, *client.QueryStreamResponse) { samples := []cortexpbv2.Sample{ { Timestamp: timestamp, @@ -2955,7 +4269,7 @@ func mockWriteRequestWithMetadataV2(t *testing.T, lbls labels.Labels, symbols [] }, } - req := cortexpbv2.ToWriteRequestV2([]labels.Labels{lbls}, symbols, samples, nil, []cortexpbv2.Metadata{metadata}, cortexpbv2.API) + req := cortexpbv2.ToWriteRequestV2([]labels.Labels{lbls}, samples, nil, []cortexpbv2.Metadata{metadata}, cortexpbv2.API, additionalSymbols...) chunk := chunkenc.NewXORChunk() app, err := chunk.Appender() @@ -3013,12 +4327,7 @@ func mockHistogramWriteRequestV2(t *testing.T, lbls labels.Labels, value int, ti require.NoError(t, err) c.Compact() - st := writev2.NewSymbolTable() - for _, lb := range lbls { - st.Symbolize(lb.Name) - st.Symbolize(lb.Value) - } - req := cortexpbv2.ToWriteRequestV2([]labels.Labels{lbls}, st.Symbols(), nil, histograms, nil, cortexpbv2.API) + req := cortexpbv2.ToWriteRequestV2([]labels.Labels{lbls}, nil, histograms, nil, cortexpbv2.API) enc := int32(encoding.PrometheusHistogramChunk) if float { enc = int32(encoding.PrometheusFloatHistogramChunk) @@ -3049,13 +4358,8 @@ func mockWriteRequestV2(t *testing.T, lbls labels.Labels, value float64, timesta Value: value, }, } - st := writev2.NewSymbolTable() - for _, lb := range lbls { - st.Symbolize(lb.Name) - st.Symbolize(lb.Value) - } - req := cortexpbv2.ToWriteRequestV2([]labels.Labels{lbls}, st.Symbols(), samples, nil, nil, cortexpbv2.API) + req := cortexpbv2.ToWriteRequestV2([]labels.Labels{lbls}, samples, nil, nil, cortexpbv2.API) chunk := chunkenc.NewXORChunk() app, err := chunk.Appender() @@ -3084,15 +4388,13 @@ func mockWriteRequestV2(t *testing.T, lbls labels.Labels, value float64, timesta func pushSingleSampleWithMetadataV2(t *testing.T, i *Ingester) { ctx := user.InjectOrgID(context.Background(), userID) - - symbols := []string{"", "__name__", "test", "a help for metric"} metadata := cortexpbv2.Metadata{ Type: cortexpbv2.METRIC_TYPE_COUNTER, HelpRef: 3, UnitRef: 0, } - req, _ := mockWriteRequestWithMetadataV2(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, symbols, 0, util.TimeToMillis(time.Now()), metadata) + req, _ := mockWriteRequestWithMetadataV2(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, util.TimeToMillis(time.Now()), metadata, "a help for metric") _, err := i.PushV2(ctx, req) require.NoError(t, err) } @@ -3152,14 +4454,10 @@ func createIngesterWithSeriesV2(t testing.TB, userID string, numSeries, numSampl metrics := make([]labels.Labels, 0, batchSize) samples := make([]cortexpbv2.Sample, 0, batchSize) - st := writev2.NewSymbolTable() - for s := 0; s < batchSize; s++ { metrics = append(metrics, labels.Labels{ {Name: labels.MetricName, Value: fmt.Sprintf("test_%d", o+s)}, }) - st.Symbolize("__name__") - st.Symbolize(fmt.Sprintf("test_%d", o+s)) samples = append(samples, cortexpbv2.Sample{ Timestamp: ts, @@ -3168,7 +4466,7 @@ func createIngesterWithSeriesV2(t testing.TB, userID string, numSeries, numSampl } // Send metrics to the ingester. - req := cortexpbv2.ToWriteRequestV2(metrics, st.Symbols(), samples, nil, nil, cortexpbv2.API) + req := cortexpbv2.ToWriteRequestV2(metrics, samples, nil, nil, cortexpbv2.API) _, err := i.PushV2(ctx, req) require.NoError(t, err) }