feat: Tokenizer support build with params and clone for concurrency

internal/core/src/common/FieldMeta.cpp

@@ -22,18 +22,9 @@ TokenizerParams
 ParseTokenizerParams(const TypeParams& params) {
     auto iter = params.find("tokenizer_params");
     if (iter == params.end()) {
-        return {};
+        return "{}";
     }
-    nlohmann::json j = nlohmann::json::parse(iter->second);
-    std::map<std::string, std::string> ret;
-    for (const auto& [k, v] : j.items()) {
-        try {
-            ret[k] = v.get<std::string>();
-        } catch (std::exception& e) {
-            ret[k] = v.dump();
-        }
-    }
-    return ret;
+    return iter->second;
 }
 
 bool

internal/core/src/common/FieldMeta.h

@@ -25,7 +25,7 @@
 
 namespace milvus {
 using TypeParams = std::map<std::string, std::string>;
-using TokenizerParams = std::map<std::string, std::string>;
+using TokenizerParams = std::string;
 
 TokenizerParams
 ParseTokenizerParams(const TypeParams& params);

internal/core/src/index/TextMatchIndex.cpp

@@ -19,10 +19,9 @@
 namespace milvus::index {
 constexpr const char* TMP_TEXT_LOG_PREFIX = "/tmp/milvus/text-log/";
 
-TextMatchIndex::TextMatchIndex(
-    int64_t commit_interval_in_ms,
-    const char* tokenizer_name,
-    const std::map<std::string, std::string>& tokenizer_params)
+TextMatchIndex::TextMatchIndex(int64_t commit_interval_in_ms,
+                               const char* tokenizer_name,
+                               const char* tokenizer_params)
     : commit_interval_in_ms_(commit_interval_in_ms),
       last_commit_time_(stdclock::now()) {
     d_type_ = TantivyDataType::Text;
@@ -31,10 +30,9 @@
         field_name.c_str(), true, "", tokenizer_name, tokenizer_params);
 }
 
-TextMatchIndex::TextMatchIndex(
-    const std::string& path,
-    const char* tokenizer_name,
-    const std::map<std::string, std::string>& tokenizer_params)
+TextMatchIndex::TextMatchIndex(const std::string& path,
+                               const char* tokenizer_name,
+                               const char* tokenizer_params)
     : commit_interval_in_ms_(std::numeric_limits<int64_t>::max()),
       last_commit_time_(stdclock::now()) {
     path_ = path;
@@ -47,10 +45,9 @@
                                                      tokenizer_params);
 }
 
-TextMatchIndex::TextMatchIndex(
-    const storage::FileManagerContext& ctx,
-    const char* tokenizer_name,
-    const std::map<std::string, std::string>& tokenizer_params)
+TextMatchIndex::TextMatchIndex(const storage::FileManagerContext& ctx,
+                               const char* tokenizer_name,
+                               const char* tokenizer_params)
     : commit_interval_in_ms_(std::numeric_limits<int64_t>::max()),
       last_commit_time_(stdclock::now()) {
     schema_ = ctx.fieldDataMeta.field_schema;
@@ -174,9 +171,8 @@
 }
 
 void
-TextMatchIndex::RegisterTokenizer(
-    const char* tokenizer_name,
-    const std::map<std::string, std::string>& tokenizer_params) {
+TextMatchIndex::RegisterTokenizer(const char* tokenizer_name,
+                                  const char* tokenizer_params) {
     wrapper_->register_tokenizer(tokenizer_name, tokenizer_params);
 }
 

internal/core/src/index/TextMatchIndex.h

@@ -22,20 +22,17 @@ using stdclock = std::chrono::high_resolution_clock;
 class TextMatchIndex : public InvertedIndexTantivy<std::string> {
  public:
     // for growing segment.
-    explicit TextMatchIndex(
-        int64_t commit_interval_in_ms,
-        const char* tokenizer_name,
-        const std::map<std::string, std::string>& tokenizer_params);
+    explicit TextMatchIndex(int64_t commit_interval_in_ms,
+                            const char* tokenizer_name,
+                            const char* tokenizer_params);
     // for sealed segment.
-    explicit TextMatchIndex(
-        const std::string& path,
-        const char* tokenizer_name,
-        const std::map<std::string, std::string>& tokenizer_params);
+    explicit TextMatchIndex(const std::string& path,
+                            const char* tokenizer_name,
+                            const char* tokenizer_params);
     // for building index.
-    explicit TextMatchIndex(
-        const storage::FileManagerContext& ctx,
-        const char* tokenizer_name,
-        const std::map<std::string, std::string>& tokenizer_params);
+    explicit TextMatchIndex(const storage::FileManagerContext& ctx,
+                            const char* tokenizer_name,
+                            const char* tokenizer_params);
     // for loading index
     explicit TextMatchIndex(const storage::FileManagerContext& ctx);
 
@@ -67,9 +64,7 @@ class TextMatchIndex : public InvertedIndexTantivy<std::string> {
     CreateReader();
 
     void
-    RegisterTokenizer(
-        const char* tokenizer_name,
-        const std::map<std::string, std::string>& tokenizer_params);
+    RegisterTokenizer(const char* tokenizer_name, const char* tokenizer_params);
 
     TargetBitmap
     MatchQuery(const std::string& query);

internal/core/src/indexbuilder/index_c.cpp

@@ -284,7 +284,7 @@
         auto index = std::make_unique<index::TextMatchIndex>(
             fileManagerContext,
             "milvus_tokenizer",
-            field_schema.get_tokenizer_params());
+            field_schema.get_tokenizer_params().c_str());
         index->Build(config);
         auto binary =
             std::make_unique<knowhere::BinarySet>(index->Upload(config));

internal/core/src/segcore/ChunkedSegmentSealedImpl.cpp

@@ -1613,13 +1613,13 @@
         index = std::make_unique<index::TextMatchIndex>(
             std::numeric_limits<int64_t>::max(),
             "milvus_tokenizer",
-            field_meta.get_tokenizer_params());
+            field_meta.get_tokenizer_params().c_str());
     } else {
         // build text index using mmap.
         index = std::make_unique<index::TextMatchIndex>(
             cfg.GetMmapPath(),
             "milvus_tokenizer",
-            field_meta.get_tokenizer_params());
+            field_meta.get_tokenizer_params().c_str());
     }
 
     {
@@ -1669,7 +1669,7 @@
     index->Reload();
 
     index->RegisterTokenizer("milvus_tokenizer",
-                             field_meta.get_tokenizer_params());
+                             field_meta.get_tokenizer_params().c_str());
 
     text_indexes_[field_id] = std::move(index);
 }
@@ -1680,7 +1680,7 @@
     std::unique_lock lck(mutex_);
     const auto& field_meta = schema_->operator[](field_id);
     index->RegisterTokenizer("milvus_tokenizer",
-                             field_meta.get_tokenizer_params());
+                             field_meta.get_tokenizer_params().c_str());
     text_indexes_[field_id] = std::move(index);
 }
 

internal/core/src/segcore/SegmentGrowingImpl.cpp

@@ -859,11 +859,11 @@ SegmentGrowingImpl::CreateTextIndex(FieldId field_id) {
                "cannot create text index on non-string type");
     // todo: make this(200) configurable.
     auto index = std::make_unique<index::TextMatchIndex>(
-        200, "milvus_tokenizer", field_meta.get_tokenizer_params());
+        200, "milvus_tokenizer", field_meta.get_tokenizer_params().c_str());
     index->Commit();
     index->CreateReader();
     index->RegisterTokenizer("milvus_tokenizer",
-                             field_meta.get_tokenizer_params());
+                             field_meta.get_tokenizer_params().c_str());
     text_indexes_[field_id] = std::move(index);
 }
 

internal/core/src/segcore/SegmentSealedImpl.cpp

@@ -2043,13 +2043,13 @@
         index = std::make_unique<index::TextMatchIndex>(
             std::numeric_limits<int64_t>::max(),
             "milvus_tokenizer",
-            field_meta.get_tokenizer_params());
+            field_meta.get_tokenizer_params().c_str());
     } else {
         // build text index using mmap.
         index = std::make_unique<index::TextMatchIndex>(
             cfg.GetMmapPath(),
             "milvus_tokenizer",
-            field_meta.get_tokenizer_params());
+            field_meta.get_tokenizer_params().c_str());
     }
 
     {
@@ -2098,7 +2098,7 @@
     index->Reload();
 
     index->RegisterTokenizer("milvus_tokenizer",
-                             field_meta.get_tokenizer_params());
+                             field_meta.get_tokenizer_params().c_str());
 
     text_indexes_[field_id] = std::move(index);
 }
@@ -2109,7 +2109,7 @@
     std::unique_lock lck(mutex_);
     const auto& field_meta = schema_->operator[](field_id);
     index->RegisterTokenizer("milvus_tokenizer",
-                             field_meta.get_tokenizer_params());
+                             field_meta.get_tokenizer_params().c_str());
     text_indexes_[field_id] = std::move(index);
 }
 

internal/core/src/segcore/tokenizer_c.cpp

@@ -10,6 +10,7 @@
 // or implied. See the License for the specific language governing permissions and limitations under the License
 
 #include "segcore/tokenizer_c.h"
+#include <memory>
 #include "common/FieldMeta.h"
 #include "common/protobuf_utils.h"
 #include "pb/schema.pb.h"
@@ -19,17 +20,27 @@
 using Map = std::map<std::string, std::string>;
 
 CStatus
-create_tokenizer(CMap m, CTokenizer* tokenizer) {
+create_tokenizer(const char* params, CTokenizer* tokenizer) {
     try {
-        auto mm = reinterpret_cast<Map*>(m);
-        auto impl = std::make_unique<milvus::tantivy::Tokenizer>(*mm);
+        auto impl = std::make_unique<milvus::tantivy::Tokenizer>(params);
         *tokenizer = impl.release();
         return milvus::SuccessCStatus();
     } catch (std::exception& e) {
         return milvus::FailureCStatus(&e);
     }
 }
 
+CStatus
+clone_tokenizer(CTokenizer* tokenizer, CTokenizer* rst) {
+    try {
+        auto impl = reinterpret_cast<milvus::tantivy::Tokenizer*>(*tokenizer);
+        *rst = impl->Clone().release();
+        return milvus::SuccessCStatus();
+    } catch (std::exception& e) {
+        return milvus::FailureCStatus(&e);
+    }
+}
+
 void
 free_tokenizer(CTokenizer tokenizer) {
     auto impl = reinterpret_cast<milvus::tantivy::Tokenizer*>(tokenizer);

internal/core/src/segcore/tokenizer_c.h

@@ -24,7 +24,10 @@ extern "C" {
 typedef void* CTokenizer;
 
 CStatus
-create_tokenizer(CMap m, CTokenizer* tokenizer);
+create_tokenizer(const char* params, CTokenizer* tokenizer);
+
+CStatus
+clone_tokenizer(CTokenizer* tokenizer, CTokenizer* rst);
 
 void
 free_tokenizer(CTokenizer tokenizer);

internal/core/thirdparty/tantivy/tantivy-binding/Cargo.toml

@@ -15,6 +15,7 @@ env_logger = "0.11.3"
 log = "0.4.21"
 tantivy-jieba = "0.10.0"
 lazy_static = "1.4.0"
+serde_json = "1.0.128"
 
 [build-dependencies]
 cbindgen = "0.26.0"

internal/core/thirdparty/tantivy/tantivy-binding/include/tantivy-binding.h

@@ -88,7 +88,9 @@ RustArray tantivy_regex_query(void *ptr, const char *pattern);
 
 RustArray tantivy_match_query(void *ptr, const char *query);
 
-void tantivy_register_tokenizer(void *ptr, const char *tokenizer_name, void *tokenizer_params);
+void tantivy_register_tokenizer(void *ptr,
+                                const char *tokenizer_name,
+                                const char *tokenizer_params);
 
 void *tantivy_create_index(const char *field_name,
                            TantivyDataType data_type,
@@ -142,7 +144,7 @@ void tantivy_index_add_multi_keywords(void *ptr,
 void *tantivy_create_text_writer(const char *field_name,
                                  const char *path,
                                  const char *tokenizer_name,
-                                 void *tokenizer_params,
+                                 const char *tokenizer_params,
                                  uintptr_t num_threads,
                                  uintptr_t overall_memory_budget_in_bytes,
                                  bool in_ram);
@@ -157,7 +159,9 @@ bool tantivy_token_stream_advance(void *token_stream);
 
 const char *tantivy_token_stream_get_token(void *token_stream);
 
-void *tantivy_create_tokenizer(void *tokenizer_params);
+void *tantivy_create_tokenizer(const char *tokenizer_params);
+
+void *tantivy_clone_tokenizer(void *ptr);
 
 void tantivy_free_tokenizer(void *tokenizer);
 

internal/core/thirdparty/tantivy/tantivy-binding/src/error.rs

@@ -0,0 +1,40 @@
+use serde_json as json;
+
+#[derive(Debug)]
+pub struct TantivyError{
+    reason: String,
+}
+
+impl TantivyError{
+    fn new(reason:String) -> Self{
+        TantivyError{reason:reason}
+    }
+
+    pub fn reason(&self) -> String{
+        return self.reason.clone()
+    }
+}
+
+impl From<&str> for TantivyError{
+    fn from(value: &str) -> Self {
+        Self::new(value.to_string())
+    }
+}
+
+impl From<String> for TantivyError{
+    fn from(value: String) -> Self {
+        Self::new(value)
+    }
+}
+
+impl From<json::Error> for TantivyError{
+    fn from(value: json::Error) -> Self {
+        Self::new(value.to_string())
+    }
+}
+
+impl ToString for TantivyError{
+    fn to_string(&self) -> String {
+        return self.reason()
+    }
+}

internal/core/thirdparty/tantivy/tantivy-binding/src/index_reader_text.rs

@@ -4,7 +4,7 @@ use tantivy::{
     Term,
 };
 
-use crate::{index_reader::IndexReaderWrapper, tokenizer::default_tokenizer};
+use crate::{index_reader::IndexReaderWrapper, tokenizer::standard_analyzer};
 
 impl IndexReaderWrapper {
     // split the query string into multiple tokens using index's default tokenizer,
@@ -14,7 +14,7 @@ impl IndexReaderWrapper {
         let mut tokenizer = self
             .index
             .tokenizer_for_field(self.field)
-            .unwrap_or(default_tokenizer())
+            .unwrap_or(standard_analyzer(vec![]))
             .clone();
         let mut token_stream = tokenizer.token_stream(q);
         let mut terms: Vec<Term> = Vec::new();