refine ttl expire logic

Signed-off-by: yangxuan <[email protected]>
milvus-io · Dec 23, 2024 · 92b09c8 · 92b09c8
1 parent 4e2327a
commit 92b09c8
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 25 deletions.
diff --git a/internal/datanode/compaction/compactor_common.go b/internal/datanode/compaction/compactor_common.go
@@ -19,7 +19,6 @@ package compaction
 import (
 	"context"
 	sio "io"
-	"math"
 	"strconv"
 	"time"
 
@@ -41,7 +40,7 @@ const compactionBatchSize = 100
 
 type EntityFilter struct {
 	deletedPkTs map[interface{}]*tsHit // pk2ts
-	ttl         int64
+	ttl         int64                  // nanoseconds
 	currentTime time.Time
 
 	expiredCount int
@@ -105,31 +104,19 @@ func (filter *EntityFilter) isEntityDeleted(pk interface{}, ts typeutil.Timestam
 	return false
 }
 
-// Largest TTL is math.MaxInt64
 func (filter *EntityFilter) isEntityExpired(entityTs typeutil.Timestamp) bool {
 	// entity expire is not enabled if duration <= 0
 	if filter.ttl <= 0 {
 		return false
 	}
-
 	entityTime, _ := tsoutil.ParseTS(entityTs)
 
-	// Unlikely to happen, but avoid following negative duration
-	if entityTime.After(filter.currentTime) {
-		return false
-	}
-
-	gotDur := filter.currentTime.Sub(entityTime)
-	if gotDur == time.Duration(math.MaxInt64) {
-		// Aviod using time.Duration if duration between
-		// entityTime and currentTime is larger than max dur(~290years)
-		if filter.ttl == math.MaxInt64 {
-			return true
-		}
-		return false
-	}
+	// this dur can represents 292 million years before or after 1970, enough for milvus
+	// ttl calculation
+	dur := filter.currentTime.UnixMilli() - entityTime.UnixMilli()
 
-	return int64(gotDur.Seconds()) >= filter.ttl
+	// filter.ttl is nanoseconds
+	return filter.ttl/int64(time.Millisecond) <= dur
 }
 
 type tsHit struct {

diff --git a/internal/datanode/compaction/compactor_common_test.go b/internal/datanode/compaction/compactor_common_test.go
@@ -45,7 +45,7 @@ func (s *CompactorCommonSuite) TestEntityFilterByTTL() {
 
 		expect bool
 	}{
-		// ttl == maxInt64, no entities should expire
+		// ttl == maxInt64, dur is 1hour, no entities should expire
 		{"ttl=maxInt64, now<entity", math.MaxInt64, milvusBirthday, milvusBirthday.Add(time.Hour), false},
 		{"ttl=maxInt64, now>entity", math.MaxInt64, milvusBirthday, milvusBirthday.Add(-time.Hour), false},
 		{"ttl=maxInt64, now==entity", math.MaxInt64, milvusBirthday, milvusBirthday, false},
@@ -54,15 +54,15 @@ func (s *CompactorCommonSuite) TestEntityFilterByTTL() {
 		{"ttl=0, now>entity", 0, milvusBirthday, milvusBirthday.Add(-time.Hour), false},
 		{"ttl=0, now<entity", 0, milvusBirthday, milvusBirthday.Add(time.Hour), false},
 		// ttl == 10days
-		{"ttl=10days, nowTs-entityTs>10days", 864000, milvusBirthday.AddDate(0, 0, 11), milvusBirthday, true},
-		{"ttl=10days, nowTs-entityTs==10days", 864000, milvusBirthday.AddDate(0, 0, 10), milvusBirthday, true},
-		{"ttl=10days, nowTs-entityTs<10days", 864000, milvusBirthday.AddDate(0, 0, 9), milvusBirthday, false},
+		{"ttl=10days, nowTs-entityTs>10days", 864000000000000, milvusBirthday.AddDate(0, 0, 11), milvusBirthday, true},
+		{"ttl=10days, nowTs-entityTs==10days", 864000000000000, milvusBirthday.AddDate(0, 0, 10), milvusBirthday, true},
+		{"ttl=10days, nowTs-entityTs<10days", 864000000000000, milvusBirthday.AddDate(0, 0, 9), milvusBirthday, false},
 		// ttl is maxInt64
 		{"ttl=maxInt64, nowTs-entityTs>1000years", math.MaxInt64, milvusBirthday.AddDate(1000, 0, 11), milvusBirthday, true},
 		{"ttl=maxInt64, nowTs-entityTs==1000years", math.MaxInt64, milvusBirthday.AddDate(1000, 0, 0), milvusBirthday, true},
-		{"ttl<maxInt64, nowTs-entityTs==1000years", math.MaxInt64 - 1, milvusBirthday.AddDate(1000, 0, 0), milvusBirthday, false},
 		{"ttl=maxInt64, nowTs-entityTs==240year", math.MaxInt64, milvusBirthday.AddDate(240, 0, 0), milvusBirthday, false},
 		{"ttl=maxInt64, nowTs-entityTs==maxDur", math.MaxInt64, milvusBirthday.Add(time.Duration(math.MaxInt64)), milvusBirthday, true},
+		{"ttl<maxInt64, nowTs-entityTs==1000years", math.MaxInt64 - 1, milvusBirthday.AddDate(1000, 0, 0), milvusBirthday, true},
 	}
 	for _, test := range tests {
 		s.Run(test.description, func() {

diff --git a/internal/proto/data_coord.proto b/internal/proto/data_coord.proto
@@ -604,7 +604,7 @@ message CompactionPlan {
   CompactionType type = 5;
   uint64 timetravel = 6;
   string channel = 7;
-  int64 collection_ttl = 8;
+  int64 collection_ttl = 8; // nanoseconds
   int64 total_rows = 9;
   schema.CollectionSchema schema = 10;
   int64 clustering_key_field = 11;