From 7d4ac34b26960c825bbaff4604758630b5e3366c Mon Sep 17 00:00:00 2001 From: Yi Jin Date: Tue, 26 Mar 2024 22:15:48 -0700 Subject: [PATCH] [PLAT-104290] must wrap error as retryable to avoid compactor error exit Signed-off-by: Yi Jin --- pkg/compact/overlapping.go | 11 ++++++++--- pkg/compact/overlapping_test.go | 8 ++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/compact/overlapping.go b/pkg/compact/overlapping.go index 8db34ca398..32badf425a 100644 --- a/pkg/compact/overlapping.go +++ b/pkg/compact/overlapping.go @@ -97,14 +97,19 @@ func (o OverlappingCompactionLifecycleCallback) PreCompactionCallback(ctx contex outer = currB prev = curr } - reason = fmt.Errorf("found full overlapping block: %s > %s", outer.String(), inner.String()) + if outer.Thanos.Source == metadata.ReceiveSource { + level.Warn(logger).Log("msg", "bypass if larger blocks are from receive", + "outer", outer.String(), "inner", inner.String()) + continue + } + reason = retry(fmt.Errorf("found full overlapping block: %s > %s", outer.String(), inner.String())) if err := block.MarkForNoCompact(ctx, logger, cg.bkt, inner.ULID, overlappingReason, reason.Error(), o.noCompaction); err != nil { - return retry(err) + return err } if err := block.MarkForNoDownsample(ctx, logger, cg.bkt, inner.ULID, overlappingReason, reason.Error(), o.noDownsampling); err != nil { - return retry(err) + return err } } return reason diff --git a/pkg/compact/overlapping_test.go b/pkg/compact/overlapping_test.go index a595b109d6..279222264e 100644 --- a/pkg/compact/overlapping_test.go +++ b/pkg/compact/overlapping_test.go @@ -72,7 +72,6 @@ func TestPreCompactionCallback(t *testing.T) { createCustomBlockMeta(7, 2, 7, metadata.CompactorSource, 1), createCustomBlockMeta(8, 2, 8, metadata.ReceiveSource, 1), }, - expectedMarks: map[int]int{7: 2}, }, { testName: "full overlapping blocks", @@ -116,10 +115,15 @@ func TestPreCompactionCallback(t *testing.T) { bkt := objstore.NewInMemBucket() group := &Group{logger: log.NewNopLogger(), bkt: bkt} err := callback.PreCompactionCallback(ctx, logger, group, tcase.input) - if tcase.expectedErr != nil || len(tcase.expectedMarks) != 0 { + if len(tcase.expectedMarks) != 0 { testutil.NotOk(t, err) + testutil.Assert(t, IsRetryError(err)) + } else if tcase.expectedErr != nil { + testutil.NotOk(t, err) + testutil.Assert(t, IsHaltError(err)) } else { testutil.Ok(t, err) + testutil.Assert(t, err == nil) } objs := bkt.Objects() expectedSize := 0