[PLAT-111995] detect and prevent polluted data to return to user

Signed-off-by: Yi Jin <[email protected]>

pkg/receive/multitsdb.go

@@ -662,6 +662,7 @@ func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant
 			shipper.DefaultMetaFilename,
 		)
 	}
+	s.ForceHeadMMap()
 	tenant.set(store.NewTSDBStore(logger, s, component.Receive, lset), s, ship, exemplars.NewTSDB(s, lset))
 	level.Info(logger).Log("msg", "TSDB is now ready")
 	return nil

pkg/store/detector.go

@@ -0,0 +1,32 @@
+// Copyright (c) The Thanos Authors.
+// Licensed under the Apache License 2.0.
+
+package store
+
+import (
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/thanos-io/thanos/pkg/store/storepb"
+	"strings"
+)
+
+// So far we found this is a bug in prometheus tsdb code when OOO is enabled.
+// This is a workaround to detect the issue when tsdb selects irrelevant matched data.
+// See https://github.com/thanos-io/thanos/issues/7481
+func detectCorruptLabels(labels labels.Labels, matchers []storepb.LabelMatcher) bool {
+	for _, m := range matchers {
+		v := labels.Get(m.Name)
+		if (m.Type == storepb.LabelMatcher_EQ && v != m.Value) ||
+			(m.Type == storepb.LabelMatcher_NEQ && v == m.Value) {
+			return true
+		}
+	}
+	return false
+}
+
+func requestMatches(matchers []storepb.LabelMatcher) string {
+	var b strings.Builder
+	for _, m := range matchers {
+		b.WriteString(m.String())
+	}
+	return b.String()
+}

pkg/store/tsdb.go

@@ -216,6 +216,10 @@ func (s *TSDBStore) Series(r *storepb.SeriesRequest, seriesSrv storepb.Store_Ser
 		if !shardMatcher.MatchesLabels(completeLabelset) {
 			continue
 		}
+		if detectCorruptLabels(series.Labels(), r.Matchers) {
+			return status.Errorf(codes.DataLoss, "corrupt prometheus tsdb index detected: requesting %s, got unmatched series %s",
+				requestMatches(r.Matchers), series.Labels().String())
+		}
 
 		storeSeries := storepb.Series{Labels: labelpb.ZLabelsFromPromLabels(completeLabelset)}
 		if r.SkipChunks {