From 49ef097bc3961b298d94eea71af92632672e663c Mon Sep 17 00:00:00 2001 From: congqixia Date: Tue, 26 Sep 2023 20:04:05 +0800 Subject: [PATCH] Make garbage collector fetch meta after list from storage (#27205) (#27369) Signed-off-by: Congqi Xia --- internal/datacoord/garbage_collector.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/datacoord/garbage_collector.go b/internal/datacoord/garbage_collector.go index bcc48bb109de1..a8338fda4bf8e 100644 --- a/internal/datacoord/garbage_collector.go +++ b/internal/datacoord/garbage_collector.go @@ -127,16 +127,18 @@ func (gc *garbageCollector) scan() { total = 0 valid = 0 missing = 0 - - segmentMap = typeutil.NewUniqueSet() - filesMap = typeutil.NewSet[string]() ) - segments := gc.meta.GetAllSegmentsUnsafe() - for _, segment := range segments { - segmentMap.Insert(segment.GetID()) - for _, log := range getLogs(segment) { - filesMap.Insert(log.GetLogPath()) + getMetaMap := func() (typeutil.UniqueSet, typeutil.Set[string]) { + segmentMap := typeutil.NewUniqueSet() + filesMap := typeutil.NewSet[string]() + segments := gc.meta.GetAllSegmentsUnsafe() + for _, segment := range segments { + segmentMap.Insert(segment.GetID()) + for _, log := range getLogs(segment) { + filesMap.Insert(log.GetLogPath()) + } } + return segmentMap, filesMap } // walk only data cluster related prefixes @@ -155,6 +157,7 @@ func (gc *garbageCollector) scan() { zap.String("error", err.Error()), ) } + segmentMap, filesMap := getMetaMap() log.Info("gc scan finish list object", zap.String("prefix", prefix), zap.Duration("time spent", time.Since(startTs)), zap.Int("keys", len(infoKeys))) for i, infoKey := range infoKeys { total++