fix: Clear channelcp meta and metrics ASAP (milvus-io#35658)

See also: milvus-io#35588 --------- Signed-off-by: yangxuan <[email protected]>
bigsheeper · Aug 26, 2024 · 82743c5 · 82743c5
1 parent f756f01
commit 82743c5
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 3 deletions.
diff --git a/internal/datacoord/garbage_collector.go b/internal/datacoord/garbage_collector.go
@@ -510,9 +510,12 @@ func (gc *garbageCollector) recycleChannelCPMeta(ctx context.Context) {
 			continue
 		}
 
-		if err := gc.meta.DropChannelCheckpoint(vChannel); err != nil {
+		err := gc.meta.DropChannelCheckpoint(vChannel)
+		if err != nil {
 			// Try to GC in the next gc cycle if drop channel cp meta fail.
-			log.Warn("failed to drop channel check point during gc", zap.String("vchannel", vChannel), zap.Error(err))
+			log.Warn("failed to drop channelcp check point during gc", zap.String("vchannel", vChannel), zap.Error(err))
+		} else {
+			log.Info("GC channel cp", zap.String("vchannel", vChannel))
 		}
 	}
 

diff --git a/internal/datacoord/handler.go b/internal/datacoord/handler.go
@@ -458,7 +458,12 @@ func (h *ServerHandler) FinishDropChannel(channel string, collectionID int64) er
 		log.Warn("DropChannel failed", zap.String("vChannel", channel), zap.Error(err))
 		return err
 	}
-	log.Info("DropChannel succeeded", zap.String("vChannel", channel))
+	err = h.s.meta.DropChannelCheckpoint(channel)
+	if err != nil {
+		log.Warn("DropChannel failed to drop channel checkpoint", zap.String("channel", channel), zap.Error(err))
+		return err
+	}
+	log.Info("DropChannel succeeded", zap.String("channel", channel))
 	// Channel checkpoints are cleaned up during garbage collection.
 
 	// clean collection info cache when meet drop collection info

diff --git a/internal/datacoord/meta.go b/internal/datacoord/meta.go
@@ -1727,6 +1727,7 @@ func (m *meta) DropChannelCheckpoint(vChannel string) error {
 		return err
 	}
 	delete(m.channelCPs.checkpoints, vChannel)
+	metrics.DataCoordCheckpointUnixSeconds.DeleteLabelValues(fmt.Sprint(paramtable.GetNodeID()), vChannel)
 	log.Info("DropChannelCheckpoint done", zap.String("vChannel", vChannel))
 	return nil
 }