dynamic index version control (#27335)

Co-authored-by: longjiquan <[email protected]>
milvus-io · Sep 25, 2023 · 5db4a04 · 5db4a04
1 parent cec7e99
commit 5db4a04
Show file tree

Hide file tree

Showing 50 changed files with 1,013 additions and 693 deletions.
diff --git a/configs/milvus.yaml b/configs/milvus.yaml
@@ -523,8 +523,6 @@ common:
       info: 500 # minimum milliseconds for printing durations in info level
       warn: 1000 # minimum milliseconds for printing durations in warn level
 
-  indexEngineVersion: "knowhere-v0"
-
 # QuotaConfig, configurations of Milvus quota and limits.
 # By default, we enable:
 #   1. TT protection;

diff --git a/internal/core/src/common/Types.h b/internal/core/src/common/Types.h
@@ -165,6 +165,7 @@ using BinarySet = knowhere::BinarySet;
 using Dataset = knowhere::DataSet;
 using DatasetPtr = knowhere::DataSetPtr;
 using MetricType = knowhere::MetricType;
+using IndexVersion = knowhere::IndexVersion;
 // TODO :: type define milvus index type(vector index type and scalar index type)
 using IndexType = knowhere::IndexType;
 

diff --git a/internal/core/src/config/ConfigKnowhere.cpp b/internal/core/src/config/ConfigKnowhere.cpp
@@ -21,6 +21,7 @@
 #include "glog/logging.h"
 #include "log/Log.h"
 #include "knowhere/comp/knowhere_config.h"
+#include "knowhere/version.h"
 
 namespace milvus::config {
 
@@ -85,4 +86,14 @@ KnowhereInitSearchThreadPool(const uint32_t num_threads) {
     }
 }
 
+int32_t
+GetMinimalIndexVersion() {
+    return knowhere::Version::GetMinimalVersion().VersionNumber();
+}
+
+int32_t
+GetCurrentIndexVersion() {
+    return knowhere::Version::GetCurrentVersion().VersionNumber();
+}
+
 }  // namespace milvus::config
diff --git a/internal/core/src/config/ConfigKnowhere.h b/internal/core/src/config/ConfigKnowhere.h
@@ -30,4 +30,11 @@ KnowhereInitBuildThreadPool(const uint32_t);
 
 void
 KnowhereInitSearchThreadPool(const uint32_t);
+
+int32_t
+GetMinimalIndexVersion();
+
+int32_t
+GetCurrentIndexVersion();
+
 }  // namespace milvus::config
diff --git a/internal/core/src/index/IndexInfo.h b/internal/core/src/index/IndexInfo.h
@@ -24,7 +24,7 @@ struct CreateIndexInfo {
     DataType field_type;
     IndexType index_type;
     MetricType metric_type;
-    std::string index_engine_version;
+    IndexVersion index_engine_version;
 };
 
 }  // namespace milvus::index
diff --git a/internal/core/src/index/Utils.cpp b/internal/core/src/index/Utils.cpp
@@ -120,11 +120,13 @@ GetIndexTypeFromConfig(const Config& config) {
     return index_type.value();
 }
 
-std::string
+IndexVersion
 GetIndexEngineVersionFromConfig(const Config& config) {
     auto index_engine_version =
         GetValueFromConfig<std::string>(config, INDEX_ENGINE_VERSION);
-    return index_engine_version.has_value() ? index_engine_version.value() : "";
+    AssertInfo(index_engine_version.has_value(),
+               "index_engine not exist in config");
+    return (std::stoi(index_engine_version.value()));
 }
 
 // TODO :: too ugly

diff --git a/internal/core/src/index/Utils.h b/internal/core/src/index/Utils.h
@@ -100,7 +100,7 @@ GetMetricTypeFromConfig(const Config& config);
 std::string
 GetIndexTypeFromConfig(const Config& config);
 
-std::string
+IndexVersion
 GetIndexEngineVersionFromConfig(const Config& config);
 
 storage::FieldDataMeta

diff --git a/internal/core/src/index/VectorDiskIndex.cpp b/internal/core/src/index/VectorDiskIndex.cpp
@@ -36,7 +36,7 @@ template <typename T>
 VectorDiskAnnIndex<T>::VectorDiskAnnIndex(
     const IndexType& index_type,
     const MetricType& metric_type,
-    const std::string& version,
+    const IndexVersion& version,
     const storage::FileManagerContext& file_manager_context)
     : VectorIndex(index_type, metric_type) {
     file_manager_ =

diff --git a/internal/core/src/index/VectorDiskIndex.h b/internal/core/src/index/VectorDiskIndex.h
@@ -30,7 +30,7 @@ class VectorDiskAnnIndex : public VectorIndex {
     explicit VectorDiskAnnIndex(
         const IndexType& index_type,
         const MetricType& metric_type,
-        const std::string& version,
+        const IndexVersion& version,
         const storage::FileManagerContext& file_manager_context);
     BinarySet
     Serialize(const Config& config) override {  // deprecated

diff --git a/internal/core/src/index/VectorIndex.h b/internal/core/src/index/VectorIndex.h
@@ -91,11 +91,12 @@ class VectorIndex : public IndexBase {
     }
 
     void
-    CheckCompatible(const std::string& version) {
+    CheckCompatible(const IndexVersion& version) {
         std::string err_msg =
-            "version not support : " + version +
+            "version not support : " + std::to_string(version) +
             " , knowhere current version " +
-            knowhere::Version::GetCurrentVersion().VersionCode();
+            std::to_string(
+                knowhere::Version::GetCurrentVersion().VersionNumber());
         AssertInfo(
             knowhere::Version::VersionSupport(knowhere::Version(version)),
             err_msg);

diff --git a/internal/core/src/index/VectorMemIndex.cpp b/internal/core/src/index/VectorMemIndex.cpp
@@ -51,7 +51,7 @@ namespace milvus::index {
 VectorMemIndex::VectorMemIndex(
     const IndexType& index_type,
     const MetricType& metric_type,
-    const std::string& version,
+    const IndexVersion& version,
     const storage::FileManagerContext& file_manager_context)
     : VectorIndex(index_type, metric_type) {
     AssertInfo(!is_unsupported(index_type, metric_type),

diff --git a/internal/core/src/index/VectorMemIndex.h b/internal/core/src/index/VectorMemIndex.h
@@ -32,7 +32,7 @@ class VectorMemIndex : public VectorIndex {
     explicit VectorMemIndex(
         const IndexType& index_type,
         const MetricType& metric_type,
-        const std::string& version,
+        const IndexVersion& version,
         const storage::FileManagerContext& file_manager_context =
             storage::FileManagerContext());
 

diff --git a/internal/core/src/indexbuilder/index_c.cpp b/internal/core/src/indexbuilder/index_c.cpp
@@ -53,8 +53,8 @@ CreateIndex(enum CDataType dtype,
             config[param.key()] = param.value();
         }
 
-        config[milvus::index::INDEX_ENGINE_VERSION] =
-            knowhere::Version::GetDefaultVersion().VersionCode();
+        config[milvus::index::INDEX_ENGINE_VERSION] = std::to_string(
+            knowhere::Version::GetCurrentVersion().VersionNumber());
 
         auto& index_factory = milvus::indexbuilder::IndexFactory::GetInstance();
         auto index =
@@ -90,13 +90,11 @@ CreateIndexV2(CIndex* res_index, CBuildIndexInfo c_build_index_info) {
         AssertInfo(index_type.has_value(), "index type is empty");
         index_info.index_type = index_type.value();
 
-        auto engine_version =
-            build_index_info->index_engine_version.empty()
-                ? knowhere::Version::GetDefaultVersion().VersionCode()
-                : build_index_info->index_engine_version;
+        auto engine_version = build_index_info->index_engine_version;
 
         index_info.index_engine_version = engine_version;
-        config[milvus::index::INDEX_ENGINE_VERSION] = engine_version;
+        config[milvus::index::INDEX_ENGINE_VERSION] =
+            std::to_string(engine_version);
 
         // get metric type
         if (milvus::datatype_is_vector(field_type)) {
@@ -459,10 +457,9 @@ AppendInsertFilePath(CBuildIndexInfo c_build_index_info,
 
 CStatus
 AppendIndexEngineVersionToBuildInfo(CBuildIndexInfo c_load_index_info,
-                                    const char* c_index_engine_version) {
+                                    int32_t index_engine_version) {
     try {
         auto build_index_info = (BuildIndexInfo*)c_load_index_info;
-        std::string index_engine_version(c_index_engine_version);
         build_index_info->index_engine_version = index_engine_version;
 
         auto status = CStatus();

diff --git a/internal/core/src/indexbuilder/index_c.h b/internal/core/src/indexbuilder/index_c.h
@@ -88,7 +88,7 @@ AppendInsertFilePath(CBuildIndexInfo c_build_index_info, const char* file_path);
 
 CStatus
 AppendIndexEngineVersionToBuildInfo(CBuildIndexInfo c_load_index_info,
-                                    const char* c_index_engine_version);
+                                    int32_t c_index_engine_version);
 
 CStatus
 CreateIndexV2(CIndex* res_index, CBuildIndexInfo c_build_index_info);

diff --git a/internal/core/src/indexbuilder/types.h b/internal/core/src/indexbuilder/types.h
@@ -33,5 +33,5 @@ struct BuildIndexInfo {
     std::vector<std::string> insert_files;
     milvus::storage::StorageConfig storage_config;
     milvus::Config config;
-    std::string index_engine_version;
+    int32_t index_engine_version;
 };
diff --git a/internal/core/src/segcore/FieldIndexing.cpp b/internal/core/src/segcore/FieldIndexing.cpp
@@ -36,7 +36,7 @@ VectorFieldIndexing::VectorFieldIndexing(const FieldMeta& field_meta,
     index_ = std::make_unique<index::VectorMemIndex>(
         config_->GetIndexType(),
         config_->GetMetricType(),
-        knowhere::Version::GetCurrentVersion().VersionCode());
+        knowhere::Version::GetCurrentVersion().VersionNumber());
 }
 
 void
@@ -58,7 +58,7 @@ VectorFieldIndexing::BuildIndexRange(int64_t ack_beg,
         auto indexing = std::make_unique<index::VectorMemIndex>(
             knowhere::IndexEnum::INDEX_FAISS_IVFFLAT,
             knowhere::metric::L2,
-            knowhere::Version::GetCurrentVersion().VersionCode());
+            knowhere::Version::GetCurrentVersion().VersionNumber());
         auto dataset = knowhere::GenDataSet(
             source->get_size_per_chunk(), dim, chunk.data());
         indexing->BuildWithDataset(dataset, conf);

diff --git a/internal/core/src/segcore/Types.h b/internal/core/src/segcore/Types.h
@@ -42,7 +42,7 @@ struct LoadIndexInfo {
     std::map<std::string, std::string> index_params;
     std::vector<std::string> index_files;
     index::IndexBasePtr index;
-    std::string index_engine_version;
+    IndexVersion index_engine_version;
 };
 
 }  // namespace milvus::segcore
diff --git a/internal/core/src/segcore/load_index_c.cpp b/internal/core/src/segcore/load_index_c.cpp
@@ -211,10 +211,7 @@ AppendIndexV2(CLoadIndexInfo c_load_index_info) {
         auto& index_params = load_index_info->index_params;
         auto field_type = load_index_info->field_type;
 
-        auto engine_version =
-            load_index_info->index_engine_version.empty()
-                ? knowhere::Version::GetDefaultVersion().VersionCode()
-                : load_index_info->index_engine_version;
+        auto engine_version = load_index_info->index_engine_version;
 
         milvus::index::CreateIndexInfo index_info;
         index_info.field_type = load_index_info->field_type;
@@ -326,11 +323,10 @@ AppendIndexInfo(CLoadIndexInfo c_load_index_info,
 
 CStatus
 AppendIndexEngineVersionToLoadInfo(CLoadIndexInfo c_load_index_info,
-                                   const char* c_index_engine_version) {
+                                   int32_t index_engine_version) {
     try {
         auto load_index_info =
             (milvus::segcore::LoadIndexInfo*)c_load_index_info;
-        std::string index_engine_version(c_index_engine_version);
         load_index_info->index_engine_version = index_engine_version;
 
         auto status = CStatus();

diff --git a/internal/core/src/segcore/load_index_c.h b/internal/core/src/segcore/load_index_c.h
@@ -60,7 +60,7 @@ AppendIndexV2(CLoadIndexInfo c_load_index_info);
 
 CStatus
 AppendIndexEngineVersionToLoadInfo(CLoadIndexInfo c_load_index_info,
-                                   const char* c_index_engine_version);
+                                   int32_t index_engine_version);
 
 CStatus
 CleanLoadedIndex(CLoadIndexInfo c_load_index_info);

diff --git a/internal/core/src/segcore/segcore_init_c.cpp b/internal/core/src/segcore/segcore_init_c.cpp
@@ -77,4 +77,14 @@ SegcoreCloseGlog() {
     }
 }
 
+extern "C" int32_t
+GetCurrentIndexVersion() {
+    return milvus::config::GetCurrentIndexVersion();
+}
+
+extern "C" int32_t
+GetMinimalIndexVersion() {
+    return milvus::config::GetMinimalIndexVersion();
+}
+
 }  // namespace milvus::segcore
diff --git a/internal/core/src/segcore/segcore_init_c.h b/internal/core/src/segcore/segcore_init_c.h
@@ -43,6 +43,12 @@ SegcoreSetKnowhereSearchThreadPoolNum(const uint32_t num_threads);
 void
 SegcoreCloseGlog();
 
+int32_t
+GetCurrentIndexVersion();
+
+int32_t
+GetMinimalIndexVersion();
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt
@@ -11,7 +11,7 @@
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 #-------------------------------------------------------------------------------
 
-set( KNOWHERE_VERSION b724230 )
+set( KNOWHERE_VERSION 87190cf )
 set( GIT_REPOSITORY  "https://github.com/zilliztech/knowhere.git")
 if ( INDEX_ENGINE STREQUAL "cardinal" )
     set( KNOWHERE_VERSION main)

diff --git a/internal/core/unittest/bench/bench_indexbuilder.cpp b/internal/core/unittest/bench/bench_indexbuilder.cpp
@@ -61,7 +61,7 @@ IndexBuilder_build(benchmark::State& state) {
         config[param.key()] = param.value();
     }
     config[milvus::index::INDEX_ENGINE_VERSION] =
-        knowhere::Version::GetCurrentVersion().VersionCode();
+        std::to_string(knowhere::Version::GetCurrentVersion().VersionNumber());
 
     auto is_binary = state.range(2);
     auto dataset = GenDataset(NB, metric_type, is_binary);

diff --git a/internal/core/unittest/test_array_expr.cpp b/internal/core/unittest/test_array_expr.cpp
@@ -1531,7 +1531,7 @@ TEST(Expr, TestArrayInTerm) {
                  return false;
              }},
     };
-    
+
     std::string raw_plan_tmp = R"(vector_anns: <
                                     field_id: 100
                                     predicates: <