enhance: Use partitionID when delete by partitionKey

When delete by partition_key, Milvus will generates L0 segments globally. During L0 Compaction, those L0 segments will touch all partitions collection wise. Due to the false-positive rate of segment bloomfilters, L0 compactions will append false deltalogs to completed irrelevant partitions, which causes *partition deletion amplification. This PR uses partition_key to set targeted partitionID when producing deleteMsgs into MsgStreams. This'll narrow down L0 segments scope to partition level, and remove the false-positive influence collection-wise. However, due to DeleteMsg structure, we can only label one partition to one deleteMsg, so this enhancement fails if user wants to delete over 2 partition_keys in one deletion. Signed-off-by: yangxuan <[email protected]>
milvus-io · Dec 17, 2024 · fb7e0a1 · fb7e0a1
1 parent 989c66c
commit fb7e0a1
Show file tree

Hide file tree

Showing 9 changed files with 761 additions and 565 deletions.
diff --git a/internal/datacoord/services.go b/internal/datacoord/services.go
@@ -204,7 +204,6 @@ func (s *Server) AssignSegmentID(ctx context.Context, req *datapb.AssignSegmentI
 			zap.Int64("partitionID", r.GetPartitionID()),
 			zap.String("channelName", r.GetChannelName()),
 			zap.Uint32("count", r.GetCount()),
-			zap.String("segment level", r.GetLevel().String()),
 		)
 
 		// Load the collection info from Root Coordinator, if it is not found in server meta.

diff --git a/internal/proto/data_coord.proto b/internal/proto/data_coord.proto
@@ -167,7 +167,7 @@ message SegmentIDRequest {
   int64 partitionID = 4;
   bool isImport = 5;        // deprecated
   int64 importTaskID = 6;   // deprecated
-  SegmentLevel level = 7;
+  SegmentLevel level = 7; // deprecated
 }
 
 message AssignSegmentIDRequest {

diff --git a/internal/proxy/search_util.go b/internal/proxy/search_util.go
@@ -234,7 +234,7 @@ func getPartitionIDs(ctx context.Context, dbName string, collectionName string,
 
 	useRegexp := Params.ProxyCfg.PartitionNameRegexp.GetAsBool()
 
-	partitionsSet := typeutil.NewSet[int64]()
+	partitionsSet := typeutil.NewUniqueSet()
 	for _, partitionName := range partitionNames {
 		if useRegexp {
 			// Legacy feature, use partition name as regexp
@@ -259,9 +259,7 @@ func getPartitionIDs(ctx context.Context, dbName string, collectionName string,
 				// TODO change after testcase updated: return nil, merr.WrapErrPartitionNotFound(partitionName)
 				return nil, fmt.Errorf("partition name %s not found", partitionName)
 			}
-			if !partitionsSet.Contain(partitionID) {
-				partitionsSet.Insert(partitionID)
-			}
+			partitionsSet.Insert(partitionID)
 		}
 	}
 	return partitionsSet.Collect(), nil