Merge remote-tracking branch 'upstream/master' into upgrade-grpc-version

milvus-io · Dec 24, 2024 · e8cea63 · e8cea63
2 parents b35b74f + 5304707
commit e8cea63
Show file tree

Hide file tree

Showing 12 changed files with 3,185 additions and 33 deletions.
diff --git a/internal/flushcommon/syncmgr/sync_manager.go b/internal/flushcommon/syncmgr/sync_manager.go
@@ -124,6 +124,10 @@ func (mgr *syncManager) safeSubmitTask(ctx context.Context, task Task, callbacks
 
 func (mgr *syncManager) submit(ctx context.Context, key int64, task Task, callbacks ...func(error) error) *conc.Future[struct{}] {
 	handler := func(err error) error {
+		taskKey := fmt.Sprintf("%d-%d", task.SegmentID(), task.Checkpoint().GetTimestamp())
+		defer func() {
+			mgr.tasks.Remove(taskKey)
+		}()
 		if err == nil {
 			return nil
 		}

diff --git a/internal/http/webui/assets/index-Bybe0GVy.js b/internal/http/webui/assets/index-Bybe0GVy.js
diff --git a/internal/http/webui/index.html b/internal/http/webui/index.html
@@ -1,27 +1,19 @@
 <!doctype html>
 <html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <link
-      rel="icon"
-      type="image/svg+xml"
-      href="/vite.svg"
-    />
-    <meta
-      name="viewport"
-      content="width=device-width, initial-scale=1.0"
-    />
-    <link
-      rel="icon"
-      href="./assets/favicon-ADjA7Mb5.png"
-      type="image/png"
-    />
-    <meta />
-    <title>Milvus Management</title>
-    <script type="module" crossorigin src="./assets/index-D2OgaNZi.js"></script>
-    <link rel="stylesheet" crossorigin href="./assets/index-Cpi1DtPq.css">
-  </head>
-  <body>
-    <div id="root"></div>
-  </body>
-</html>
+
+<head>
+  <meta charset="UTF-8" />
+  <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <link rel="icon" href="./assets/favicon-ADjA7Mb5.png" type="image/png" />
+  <meta />
+  <title>Milvus Management</title>
+  <script type="module" crossorigin src="./assets/index-Bybe0GVy.js"></script>
+  <link rel="stylesheet" crossorigin href="./assets/index-Cpi1DtPq.css">
+</head>
+
+<body>
+  <div id="root"></div>
+</body>
+
+</html>
diff --git a/tests/python_client/base/collection_wrapper.py b/tests/python_client/base/collection_wrapper.py
@@ -58,6 +58,10 @@ def is_empty(self):
         self.flush()
         return self.collection.is_empty
 
+    @property
+    def is_empty_without_flush(self):
+        return self.collection.is_empty
+
     @property
     def num_entities(self):
         self.flush()

diff --git a/tests/python_client/base/database_wrapper.py b/tests/python_client/base/database_wrapper.py
@@ -33,3 +33,15 @@ def list_database(self, using="default", timeout=None, check_task=None, check_it
         response, is_succ = api_request([self.database.list_database, using, timeout])
         check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
         return response, check_result
+
+    def set_properties(self, db_name: str, properties: dict, using="default", timeout=None, check_task=None, check_items=None):
+        func_name = sys._getframe().f_code.co_name
+        response, is_succ = api_request([self.database.set_properties, db_name, properties, using, timeout])
+        check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
+        return response, check_result
+
+    def describe_database(self, db_name: str, using="default", timeout=None, check_task=None, check_items=None):
+        func_name = sys._getframe().f_code.co_name
+        response, is_succ = api_request([self.database.describe_database, db_name, using, timeout])
+        check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
+        return response, check_result
diff --git a/tests/python_client/base/high_level_api_wrapper.py b/tests/python_client/base/high_level_api_wrapper.py
@@ -665,6 +665,39 @@ def revoke_privilege(self, role_name, object_type, privilege, object_name, db_na
                                        object_name=object_name, db_name=db_name, **kwargs).run()
         return res, check_result
 
+    def create_privilege_group(self, privilege_group: str, check_task=None, check_items=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.milvus_client.create_privilege_group, privilege_group], **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
+
+    def drop_privilege_group(self, privilege_group: str, check_task=None, check_items=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.milvus_client.drop_privilege_group, privilege_group], **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
+
+    def list_privilege_groups(self, check_task=None, check_items=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.milvus_client.list_privilege_groups], **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
+
+    def add_privileges_to_group(self, privilege_group: str, privileges: list, check_task=None, check_items=None,
+                                **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.milvus_client.add_privileges_to_group, privilege_group, privileges], **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
+
+    def remove_privileges_from_group(self, privilege_group: str, privileges: list, check_task=None, check_items=None,
+                                     **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.milvus_client.remove_privileges_from_group, privilege_group, privileges],
+                                 **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
+
     @trace()
     def alter_index_properties(self, client, collection_name, index_name, properties, timeout=None,
                                check_task=None, check_items=None, **kwargs):
@@ -785,7 +818,3 @@ def list_databases(self, client, timeout=None, check_task=None, check_items=None
         check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
         return res, check_result
 
-
-
-
-
diff --git a/tests/python_client/base/partition_wrapper.py b/tests/python_client/base/partition_wrapper.py
@@ -38,11 +38,19 @@ def is_empty(self):
         self.flush()
         return self.partition.is_empty if self.partition else None
 
+    @property
+    def is_empty_without_flush(self):
+        return self.partition.is_empty if self.partition else None
+
     @property
     def num_entities(self):
         self.flush()
         return self.partition.num_entities if self.partition else None
 
+    @property
+    def num_entities_without_flush(self):
+        return self.partition.num_entities if self.partition else None
+
     def drop(self, check_task=None, check_items=None, **kwargs):
         timeout = kwargs.get("timeout", TIMEOUT)
         kwargs.update({"timeout": timeout})

diff --git a/tests/python_client/base/utility_wrapper.py b/tests/python_client/base/utility_wrapper.py
@@ -471,6 +471,18 @@ def role_revoke(self, object: str, object_name: str, privilege: str, db_name: st
         check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
         return res, check_result
 
+    def role_grant_v2(self, privilege: str, collection_name: str, db_name: str = None, check_task=None, check_items=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.role.grant_v2, privilege, collection_name, db_name], **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
+
+    def role_revoke_v2(self, privilege: str, collection_name: str, db_name: str = None, check_task=None, check_items=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.role.revoke_v2, privilege, collection_name, db_name], **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
+
     def role_list_grant(self, object: str, object_name: str, db_name: str = "", check_task=None, check_items=None, **kwargs):
         func_name = sys._getframe().f_code.co_name
         res, check = api_request([self.role.list_grant, object, object_name, db_name], **kwargs)
@@ -558,3 +570,33 @@ def list_indexes(self, collection_name, using="default", timeout=None, check_tas
         check_result = ResponseChecker(res, func_name, check_task, check_items, check,
                                        collection_name=collection_name, using=using, timeout=timeout, **kwargs).run()
         return res, check_result
+
+    def create_privilege_group(self, privilege_group: str, check_task=None, check_items=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.role.create_privilege_group, privilege_group], **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
+
+    def drop_privilege_group(self, privilege_group: str, check_task=None, check_items=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.role.drop_privilege_group, privilege_group], **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
+
+    def list_privilege_groups(self, check_task=None, check_items=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.role.list_privilege_groups], **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
+
+    def add_privileges_to_group(self, privilege_group: str, privileges: list, check_task=None, check_items=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.role.add_privileges_to_group, privilege_group, privileges], **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
+
+    def remove_privileges_from_group(self, privilege_group: str, privileges: list, check_task=None, check_items=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = api_request([self.role.remove_privileges_from_group, privilege_group, privileges], **kwargs)
+        check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
+        return res, check_result
diff --git a/tests/python_client/common/common_type.py b/tests/python_client/common/common_type.py
@@ -258,6 +258,29 @@
 default_diskann_search_params = {"params": {"search_list": 30}}
 default_sparse_search_params = {"metric_type": "IP", "params": {"drop_ratio_search": "0.2"}}
 default_text_sparse_search_params = {"metric_type": "BM25", "params": {}}
+built_in_privilege_groups = ["CollectionReadWrite", "CollectionReadOnly", "CollectionAdmin",
+                             "DatabaseReadWrite", "DatabaseReadOnly", "DatabaseAdmin",
+                             "ClusterReadWrite", "ClusterReadOnly", "ClusterAdmin"]
+privilege_group_privilege_dict = {"Query": False, "Search": False, "GetLoadState": False,
+                                  "GetLoadingProgress": False, "HasPartition": False, "ShowPartitions": False,
+                                  "ShowCollections": False, "ListAliases": False, "ListDatabases": False,
+                                  "DescribeDatabase": False, "DescribeAlias": False, "GetStatistics": False,
+                                  "CreateIndex": False, "DropIndex": False, "CreatePartition": False,
+                                  "DropPartition": False, "Load": False, "Release": False,
+                                  "Insert": False, "Delete": False, "Upsert": False,
+                                  "Import": False, "Flush": False, "Compaction": False,
+                                  "LoadBalance": False, "RenameCollection": False, "CreateAlias": False,
+                                  "DropAlias": False, "CreateCollection": False, "DropCollection": False,
+                                  "CreateOwnership": False, "DropOwnership": False, "SelectOwnership": False,
+                                  "ManageOwnership": False, "UpdateUser": False, "SelectUser": False,
+                                  "CreateResourceGroup": False, "DropResourceGroup": False,
+                                  "UpdateResourceGroups": False,
+                                  "DescribeResourceGroup": False, "ListResourceGroups": False, "TransferNode": False,
+                                  "TransferReplica": False, "CreateDatabase": False, "DropDatabase": False,
+                                  "AlterDatabase": False, "FlushAll": False, "ListPrivilegeGroups": False,
+                                  "CreatePrivilegeGroup": False, "DropPrivilegeGroup": False,
+                                  "OperatePrivilegeGroup": False}
+
 
 class CheckTasks:
     """ The name of the method used to check the result """

diff --git a/tests/python_client/conftest.py b/tests/python_client/conftest.py
@@ -22,8 +22,8 @@ def pytest_addoption(parser):
     parser.addoption("--host", action="store", default="localhost", help="service's ip")
     parser.addoption("--service", action="store", default="", help="service address")
     parser.addoption("--port", action="store", default=19530, help="service's port")
-    parser.addoption("--user", action="store", default="", help="user name for connection")
-    parser.addoption("--password", action="store", default="", help="password for connection")
+    parser.addoption("--user", action="store", default="root", help="user name for connection")
+    parser.addoption("--password", action="store", default="Milvus", help="password for connection")
     parser.addoption("--db_name", action="store", default="default", help="database name for connection")
     parser.addoption("--secure", type=bool, action="store", default=False, help="secure for connection")
     parser.addoption("--milvus_ns", action="store", default="chaos-testing", help="milvus_ns")

diff --git a/tests/python_client/milvus_client/test_milvus_client_alter.py b/tests/python_client/milvus_client/test_milvus_client_alter.py
@@ -70,8 +70,8 @@ def test_milvus_client_alter_index_default(self):
         client_w.load_collection(client, collection_name)
         res1 = client_w.describe_index(client, collection_name, index_name=idx_names[0])[0]
         assert res1.get('mmap.enabled', None) is None
-        error = {ct.err_code: 999,
-                 ct.err_msg: "can't alter index on loaded collection, please release the collection first"}
+        error = {ct.err_code: 104,
+                 ct.err_msg: f"can't alter index on loaded collection, please release the collection first: collection already loaded[collection={collection_name}]"}
         # 1. alter index after load
         client_w.alter_index_properties(client, collection_name, idx_names[0], properties={"mmap.enabled": True},
                                         check_task=CheckTasks.err_res, check_items=error)