Merge remote-tracking branch 'upstream/master' into fix-dropped-metric

milvus-io · Nov 6, 2024 · 982db11 · 982db11
2 parents 1c07e0e + c83b939
commit 982db11
Show file tree

Hide file tree

Showing 34 changed files with 482 additions and 296 deletions.
diff --git a/ci/jenkins/Nightly2.groovy b/ci/jenkins/Nightly2.groovy
@@ -1,4 +1,4 @@
-@Library('jenkins-shared-library@v0.62.0') _
+@Library('jenkins-shared-library@v0.67.0') _
 
 def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
 

diff --git a/ci/jenkins/PR-for-go-sdk.groovy b/ci/jenkins/PR-for-go-sdk.groovy
@@ -1,4 +1,4 @@
-@Library('jenkins-shared-library@v0.62.0') _
+@Library('jenkins-shared-library@v0.67.0') _
 
 def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
 
@@ -11,6 +11,13 @@ pipeline {
         buildDiscarder logRotator(artifactDaysToKeepStr: '30')
         preserveStashes(buildCount: 5)
         disableConcurrentBuilds(abortPrevious: true)
+        timeout(time: 6, unit: 'HOURS')
+        throttleJobProperty(
+            categories: ['go-sdk'],
+            throttleEnabled: true,
+            throttleOption: 'category'
+
+        )
     }
     agent {
         kubernetes {

diff --git a/ci/jenkins/PR.groovy b/ci/jenkins/PR.groovy
@@ -1,4 +1,4 @@
-@Library('jenkins-shared-library@v0.62.0') _
+@Library('jenkins-shared-library@v0.67.0') _
 
 def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
 def milvus_helm_chart_version = '4.2.8'
@@ -10,6 +10,13 @@ pipeline {
         buildDiscarder logRotator(artifactDaysToKeepStr: '30')
         preserveStashes(buildCount: 5)
         disableConcurrentBuilds(abortPrevious: true)
+        timeout(time: 6, unit: 'HOURS')
+        throttleJobProperty(
+            categories: ['cpu-e2e'],
+            throttleEnabled: true,
+            throttleOption: 'category'
+
+        )
     }
     agent {
         kubernetes {

diff --git a/ci/jenkins/UT-CPP.groovy b/ci/jenkins/UT-CPP.groovy
@@ -1,4 +1,4 @@
-@Library('jenkins-shared-library@v0.63.0') _
+@Library('jenkins-shared-library@v0.67.0') _
 
 def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
 def milvus_helm_chart_version = '4.2.8'
@@ -10,6 +10,13 @@ pipeline {
         buildDiscarder logRotator(artifactDaysToKeepStr: '30')
         preserveStashes(buildCount: 5)
         disableConcurrentBuilds(abortPrevious: true)
+        timeout(time: 6, unit: 'HOURS')
+        throttleJobProperty(
+            categories: ['cpp-unit-test'],
+            throttleEnabled: true,
+            throttleOption: 'category'
+
+        )
     }
     agent {
         kubernetes {
@@ -29,7 +36,7 @@ pipeline {
                 }
             }
         }
-        stage('build') {
+        stage('build & test') {
             steps {
                 container('tkn') {
                     script {
@@ -40,7 +47,9 @@ pipeline {
                                               pullRequestNumber: "$env.CHANGE_ID",
                                               make_cmd: "make clean && make USE_ASAN=ON build-cpp-with-coverage",
                                               test_entrypoint: "./scripts/run_cpp_codecov.sh",
-                                              codecov_files: "./lcov_output.info"
+                                              codecov_report_name: "cpp-unit-test",
+                                              codecov_files: "./lcov_output.info",
+                                              tekton_pipeline_timeout: '3h'
                     }
                 }
             }

diff --git a/client/collection.go b/client/collection.go
@@ -138,3 +138,12 @@ func (c *Client) RenameCollection(ctx context.Context, option RenameCollectionOp
 		return merr.CheckRPCCall(resp, err)
 	})
 }
+
+func (c *Client) AlterCollection(ctx context.Context, option AlterCollectionOption, callOptions ...grpc.CallOption) error {
+	req := option.Request()
+
+	return c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
+		resp, err := milvusService.AlterCollection(ctx, req, callOptions...)
+		return merr.CheckRPCCall(resp, err)
+	})
+}
diff --git a/client/collection_options.go b/client/collection_options.go
@@ -17,6 +17,8 @@
 package client
 
 import (
+	"fmt"
+
 	"google.golang.org/protobuf/proto"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
@@ -56,6 +58,8 @@ type createCollectionOption struct {
 	// partition key
 	numPartitions int64
 
+	indexOptions []CreateIndexOption
+
 	// is fast create collection
 	isFast bool
 	// fast creation with index
@@ -83,6 +87,21 @@ func (opt *createCollectionOption) WithVarcharPK(varcharPK bool, maxLen int) *cr
 	return opt
 }
 
+func (opt *createCollectionOption) WithIndexOptions(indexOpts ...CreateIndexOption) *createCollectionOption {
+	opt.indexOptions = append(opt.indexOptions, indexOpts...)
+	return opt
+}
+
+func (opt *createCollectionOption) WithProperty(key string, value any) *createCollectionOption {
+	opt.properties[key] = fmt.Sprintf("%v", value)
+	return opt
+}
+
+func (opt *createCollectionOption) WithConsistencyLevel(cl entity.ConsistencyLevel) *createCollectionOption {
+	opt.consistencyLevel = cl
+	return opt
+}
+
 func (opt *createCollectionOption) Request() *milvuspb.CreateCollectionRequest {
 	// fast create collection
 	if opt.isFast {
@@ -103,6 +122,7 @@ func (opt *createCollectionOption) Request() *milvuspb.CreateCollectionRequest {
 
 	var schemaBytes []byte
 	if opt.schema != nil {
+		opt.schema.WithName(opt.name)
 		schemaProto := opt.schema.ProtoMessage()
 		schemaBytes, _ = proto.Marshal(schemaProto)
 	}
@@ -144,6 +164,7 @@ func SimpleCreateCollectionOptions(name string, dim int64) *createCollectionOpti
 		dim:                  dim,
 		enabledDynamicSchema: true,
 		consistencyLevel:     entity.DefaultConsistencyLevel,
+		properties:           make(map[string]string),
 
 		isFast:     true,
 		metricType: entity.COSINE,
@@ -157,6 +178,7 @@ func NewCreateCollectionOption(name string, collectionSchema *entity.Schema) *cr
 		shardNum:         1,
 		schema:           collectionSchema,
 		consistencyLevel: entity.DefaultConsistencyLevel,
+		properties:       make(map[string]string),
 
 		metricType: entity.COSINE,
 	}
@@ -263,3 +285,28 @@ func NewRenameCollectionOption(oldName, newName string) *renameCollectionOption
 		newCollectionName: newName,
 	}
 }
+
+type AlterCollectionOption interface {
+	Request() *milvuspb.AlterCollectionRequest
+}
+
+type alterCollectionOption struct {
+	collectionName string
+	properties     map[string]string
+}
+
+func (opt *alterCollectionOption) WithProperty(key string, value any) *alterCollectionOption {
+	opt.properties[key] = fmt.Sprintf("%v", value)
+	return opt
+}
+
+func (opt *alterCollectionOption) Request() *milvuspb.AlterCollectionRequest {
+	return &milvuspb.AlterCollectionRequest{
+		CollectionName: opt.collectionName,
+		Properties:     entity.MapKvPairs(opt.properties),
+	}
+}
+
+func NewAlterCollectionOption(collection string) *alterCollectionOption {
+	return &alterCollectionOption{collectionName: collection, properties: make(map[string]string)}
+}
diff --git a/client/collection_test.go b/client/collection_test.go
@@ -30,6 +30,7 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
 	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
 	"github.com/milvus-io/milvus/client/v2/entity"
+	"github.com/milvus-io/milvus/pkg/common"
 	"github.com/milvus-io/milvus/pkg/util/merr"
 )
 
@@ -117,11 +118,20 @@ func (s *CollectionSuite) TestCreateCollectionOptions() {
 		WithField(entity.NewField().WithName("int64").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
 		WithField(entity.NewField().WithName("vector").WithDim(128).WithDataType(entity.FieldTypeFloatVector))
 
-	opt = NewCreateCollectionOption(collectionName, schema).WithShardNum(2)
+	opt = NewCreateCollectionOption(collectionName, schema).
+		WithShardNum(2).
+		WithConsistencyLevel(entity.ClEventually).
+		WithProperty(common.CollectionTTLConfigKey, 86400)
 
 	req = opt.Request()
 	s.Equal(collectionName, req.GetCollectionName())
 	s.EqualValues(2, req.GetShardsNum())
+	s.EqualValues(commonpb.ConsistencyLevel_Eventually, req.GetConsistencyLevel())
+	if s.Len(req.GetProperties(), 1) {
+		kv := req.GetProperties()[0]
+		s.Equal(common.CollectionTTLConfigKey, kv.GetKey())
+		s.Equal("86400", kv.GetValue())
+	}
 
 	collSchema = &schemapb.CollectionSchema{}
 	err = proto.Unmarshal(req.GetSchema(), collSchema)
@@ -274,6 +284,37 @@ func (s *CollectionSuite) TestRenameCollection() {
 	})
 }
 
+func (s *CollectionSuite) TestAlterCollection() {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	collName := fmt.Sprintf("test_collection_%s", s.randString(6))
+	key := s.randString(6)
+	value := s.randString(6)
+
+	s.Run("success", func() {
+		s.mock.EXPECT().AlterCollection(mock.Anything, mock.Anything).RunAndReturn(func(ctx context.Context, acr *milvuspb.AlterCollectionRequest) (*commonpb.Status, error) {
+			s.Equal(collName, acr.GetCollectionName())
+			if s.Len(acr.GetProperties(), 1) {
+				item := acr.GetProperties()[0]
+				s.Equal(key, item.GetKey())
+				s.Equal(value, item.GetValue())
+			}
+			return merr.Success(), nil
+		}).Once()
+
+		err := s.client.AlterCollection(ctx, NewAlterCollectionOption(collName).WithProperty(key, value))
+		s.NoError(err)
+	})
+
+	s.Run("failure", func() {
+		s.mock.EXPECT().AlterCollection(mock.Anything, mock.Anything).Return(nil, merr.WrapErrServiceInternal("mocked")).Once()
+
+		err := s.client.AlterCollection(ctx, NewAlterCollectionOption(collName).WithProperty(key, value))
+		s.Error(err)
+	})
+}
+
 func TestCollection(t *testing.T) {
 	suite.Run(t, new(CollectionSuite))
 }
diff --git a/client/read_options.go b/client/read_options.go
@@ -142,6 +142,11 @@ func (opt *searchOption) WithPartitions(partitionNames ...string) *searchOption
 	return opt
 }
 
+func (opt *searchOption) WithGroupByField(groupByField string) *searchOption {
+	opt.request.groupByField = groupByField
+	return opt
+}
+
 func NewSearchOption(collectionName string, limit int, vectors []entity.Vector) *searchOption {
 	return &searchOption{
 		collectionName: collectionName,

diff --git a/client/read_test.go b/client/read_test.go
@@ -45,6 +45,7 @@ func (s *ReadSuite) TestSearch() {
 		s.mock.EXPECT().Search(mock.Anything, mock.Anything).RunAndReturn(func(ctx context.Context, sr *milvuspb.SearchRequest) (*milvuspb.SearchResults, error) {
 			s.Equal(collectionName, sr.GetCollectionName())
 			s.ElementsMatch([]string{partitionName}, sr.GetPartitionNames())
+			// s.Equal(s)
 
 			return &milvuspb.SearchResults{
 				Status: merr.Success(),
@@ -71,7 +72,7 @@ func (s *ReadSuite) TestSearch() {
 			entity.FloatVector(lo.RepeatBy(128, func(_ int) float32 {
 				return rand.Float32()
 			})),
-		}).WithPartitions(partitionName))
+		}).WithPartitions(partitionName).WithGroupByField("group_by"))
 		s.NoError(err)
 	})
 

diff --git a/internal/core/src/common/Chunk.h b/internal/core/src/common/Chunk.h
@@ -133,13 +133,46 @@ class StringChunk : public Chunk {
         offsets_ = reinterpret_cast<uint64_t*>(data + null_bitmap_bytes_num);
     }
 
+    std::string_view
+    operator[](const int i) const {
+        if (i < 0 || i > row_nums_) {
+            PanicInfo(ErrorCode::OutOfRange, "index out of range");
+        }
+
+        return {data_ + offsets_[i], offsets_[i + 1] - offsets_[i]};
+    }
+
     std::pair<std::vector<std::string_view>, FixedVector<bool>>
     StringViews();
 
+    int
+    binary_search_string(std::string_view target) {
+        // only supported sorted pk
+        int left = 0;
+        int right = row_nums_ - 1;  // `right` should be num_rows_ - 1
+        int result =
+            -1;  // Initialize result to store the first occurrence index
+
+        while (left <= right) {
+            int mid = left + (right - left) / 2;
+            std::string_view midString = (*this)[mid];
+            if (midString == target) {
+                result = mid;     // Store the index of match
+                right = mid - 1;  // Continue searching in the left half
+            } else if (midString < target) {
+                // midString < target
+                left = mid + 1;
+            } else {
+                // midString > target
+                right = mid - 1;
+            }
+        }
+        return result;
+    }
+
     const char*
     ValueAt(int64_t idx) const override {
-        PanicInfo(ErrorCode::Unsupported,
-                  "StringChunk::ValueAt is not supported");
+        return (*this)[idx].data();
     }
 
     uint64_t*

diff --git a/internal/core/src/common/EasyAssert.h b/internal/core/src/common/EasyAssert.h
@@ -64,13 +64,13 @@ enum ErrorCode {
     MemAllocateFailed = 2034,
     MemAllocateSizeNotMatch = 2035,
     MmapError = 2036,
-    OutOfRange = 2037,
-    GcpNativeError = 2038,
-    KnowhereError = 2100,
+    // timeout or cancel related
+    FollyOtherException = 2037,
+    FollyCancel = 2038,
+    OutOfRange = 2039,
+    GcpNativeError = 2040,
 
-    // timeout or cancel related.
-    FollyOtherException = 2200,
-    FollyCancel = 2201
+    KnowhereError = 2099
 };
 namespace impl {
 void