fix: sparse inc mast less than uint32 max

Signed-off-by: aoiasd <[email protected]>
milvus-io · Dec 6, 2024 · 2401d30 · 2401d30
1 parent 18bef5e
commit 2401d30
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 1 deletion.
diff --git a/internal/util/function/bm25_function.go b/internal/util/function/bm25_function.go
@@ -97,7 +97,7 @@ func (v *BM25FunctionRunner) run(data []string, dst []map[uint32]float32) error
 		for tokenStream.Advance() {
 			token := tokenStream.Token()
 			// TODO More Hash Option
-			hash := typeutil.HashString2Uint32(token)
+			hash := typeutil.HashString2LessUint32(token)
 			embeddingMap[hash] += 1
 		}
 		dst[i] = embeddingMap

diff --git a/pkg/util/typeutil/hash.go b/pkg/util/typeutil/hash.go
@@ -19,6 +19,7 @@ package typeutil
 import (
 	"fmt"
 	"hash/crc32"
+	"math"
 	"strconv"
 	"strings"
 	"unsafe"
@@ -75,6 +76,16 @@ func HashString2Uint32(v string) uint32 {
 	return crc32.ChecksumIEEE([]byte(subString))
 }
 
+// HashString2LessUint32 hashing a string to uint32 but less than uint32 max
+func HashString2LessUint32(v string) uint32 {
+	subString := v
+	if len(v) > substringLengthForCRC {
+		subString = v[:substringLengthForCRC]
+	}
+
+	return crc32.ChecksumIEEE([]byte(subString)) % math.MaxUint32
+}
+
 // HashPK2Channels hash primary keys to channels
 func HashPK2Channels(primaryKeys *schemapb.IDs, shardNames []string) []uint32 {
 	numShard := uint32(len(shardNames))