From 92b09c867fb8368b9bd8e7c62e24e3111146e1fd Mon Sep 17 00:00:00 2001
From: yangxuan <xuan.yang@zilliz.com>
Date: Mon, 23 Dec 2024 17:44:17 +0800
Subject: [PATCH] refine ttl expire logic

Signed-off-by: yangxuan <xuan.yang@zilliz.com>
---
 .../datanode/compaction/compactor_common.go   | 25 +++++--------------
 .../compaction/compactor_common_test.go       | 10 ++++----
 internal/proto/data_coord.proto               |  2 +-
 3 files changed, 12 insertions(+), 25 deletions(-)

diff --git a/internal/datanode/compaction/compactor_common.go b/internal/datanode/compaction/compactor_common.go
index d577511c05664..790c4b4d4e1ba 100644
--- a/internal/datanode/compaction/compactor_common.go
+++ b/internal/datanode/compaction/compactor_common.go
@@ -19,7 +19,6 @@ package compaction
 import (
 	"context"
 	sio "io"
-	"math"
 	"strconv"
 	"time"
 
@@ -41,7 +40,7 @@ const compactionBatchSize = 100
 
 type EntityFilter struct {
 	deletedPkTs map[interface{}]*tsHit // pk2ts
-	ttl         int64
+	ttl         int64                  // nanoseconds
 	currentTime time.Time
 
 	expiredCount int
@@ -105,31 +104,19 @@ func (filter *EntityFilter) isEntityDeleted(pk interface{}, ts typeutil.Timestam
 	return false
 }
 
-// Largest TTL is math.MaxInt64
 func (filter *EntityFilter) isEntityExpired(entityTs typeutil.Timestamp) bool {
 	// entity expire is not enabled if duration <= 0
 	if filter.ttl <= 0 {
 		return false
 	}
-
 	entityTime, _ := tsoutil.ParseTS(entityTs)
 
-	// Unlikely to happen, but avoid following negative duration
-	if entityTime.After(filter.currentTime) {
-		return false
-	}
-
-	gotDur := filter.currentTime.Sub(entityTime)
-	if gotDur == time.Duration(math.MaxInt64) {
-		// Aviod using time.Duration if duration between
-		// entityTime and currentTime is larger than max dur(~290years)
-		if filter.ttl == math.MaxInt64 {
-			return true
-		}
-		return false
-	}
+	// this dur can represents 292 million years before or after 1970, enough for milvus
+	// ttl calculation
+	dur := filter.currentTime.UnixMilli() - entityTime.UnixMilli()
 
-	return int64(gotDur.Seconds()) >= filter.ttl
+	// filter.ttl is nanoseconds
+	return filter.ttl/int64(time.Millisecond) <= dur
 }
 
 type tsHit struct {
diff --git a/internal/datanode/compaction/compactor_common_test.go b/internal/datanode/compaction/compactor_common_test.go
index 6a1016ae07e11..122bdc8307eb5 100644
--- a/internal/datanode/compaction/compactor_common_test.go
+++ b/internal/datanode/compaction/compactor_common_test.go
@@ -45,7 +45,7 @@ func (s *CompactorCommonSuite) TestEntityFilterByTTL() {
 
 		expect bool
 	}{
-		// ttl == maxInt64, no entities should expire
+		// ttl == maxInt64, dur is 1hour, no entities should expire
 		{"ttl=maxInt64, now<entity", math.MaxInt64, milvusBirthday, milvusBirthday.Add(time.Hour), false},
 		{"ttl=maxInt64, now>entity", math.MaxInt64, milvusBirthday, milvusBirthday.Add(-time.Hour), false},
 		{"ttl=maxInt64, now==entity", math.MaxInt64, milvusBirthday, milvusBirthday, false},
@@ -54,15 +54,15 @@ func (s *CompactorCommonSuite) TestEntityFilterByTTL() {
 		{"ttl=0, now>entity", 0, milvusBirthday, milvusBirthday.Add(-time.Hour), false},
 		{"ttl=0, now<entity", 0, milvusBirthday, milvusBirthday.Add(time.Hour), false},
 		// ttl == 10days
-		{"ttl=10days, nowTs-entityTs>10days", 864000, milvusBirthday.AddDate(0, 0, 11), milvusBirthday, true},
-		{"ttl=10days, nowTs-entityTs==10days", 864000, milvusBirthday.AddDate(0, 0, 10), milvusBirthday, true},
-		{"ttl=10days, nowTs-entityTs<10days", 864000, milvusBirthday.AddDate(0, 0, 9), milvusBirthday, false},
+		{"ttl=10days, nowTs-entityTs>10days", 864000000000000, milvusBirthday.AddDate(0, 0, 11), milvusBirthday, true},
+		{"ttl=10days, nowTs-entityTs==10days", 864000000000000, milvusBirthday.AddDate(0, 0, 10), milvusBirthday, true},
+		{"ttl=10days, nowTs-entityTs<10days", 864000000000000, milvusBirthday.AddDate(0, 0, 9), milvusBirthday, false},
 		// ttl is maxInt64
 		{"ttl=maxInt64, nowTs-entityTs>1000years", math.MaxInt64, milvusBirthday.AddDate(1000, 0, 11), milvusBirthday, true},
 		{"ttl=maxInt64, nowTs-entityTs==1000years", math.MaxInt64, milvusBirthday.AddDate(1000, 0, 0), milvusBirthday, true},
-		{"ttl<maxInt64, nowTs-entityTs==1000years", math.MaxInt64 - 1, milvusBirthday.AddDate(1000, 0, 0), milvusBirthday, false},
 		{"ttl=maxInt64, nowTs-entityTs==240year", math.MaxInt64, milvusBirthday.AddDate(240, 0, 0), milvusBirthday, false},
 		{"ttl=maxInt64, nowTs-entityTs==maxDur", math.MaxInt64, milvusBirthday.Add(time.Duration(math.MaxInt64)), milvusBirthday, true},
+		{"ttl<maxInt64, nowTs-entityTs==1000years", math.MaxInt64 - 1, milvusBirthday.AddDate(1000, 0, 0), milvusBirthday, true},
 	}
 	for _, test := range tests {
 		s.Run(test.description, func() {
diff --git a/internal/proto/data_coord.proto b/internal/proto/data_coord.proto
index 8308c70e04416..5d78f28cf0ddf 100644
--- a/internal/proto/data_coord.proto
+++ b/internal/proto/data_coord.proto
@@ -604,7 +604,7 @@ message CompactionPlan {
   CompactionType type = 5;
   uint64 timetravel = 6;
   string channel = 7;
-  int64 collection_ttl = 8;
+  int64 collection_ttl = 8; // nanoseconds
   int64 total_rows = 9;
   schema.CollectionSchema schema = 10;
   int64 clustering_key_field = 11;