Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enhance: Handle rust error in c++ #38113

Merged
merged 8 commits into from
Dec 16, 2024
Merged

enhance: Handle rust error in c++ #38113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4972f93
enhance: Handle rust error in c++
sunby Nov 26, 2024
393094d
fix e2e
sunby Dec 2, 2024
1b41815
fix double free
sunby Dec 11, 2024
599c31a
add ut
sunby Dec 11, 2024
df2a70a
fix format
sunby Dec 12, 2024
393c379
fix format
sunby Dec 13, 2024
22c4c9c
fix
sunby Dec 16, 2024
dedd203
fix
sunby Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 46 additions & 1 deletion internal/core/thirdparty/tantivy/rust-array.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#pragma once

#include <iostream>
#include <memory>
#include <sstream>

#include "tantivy-binding.h"
Expand All @@ -11,7 +12,13 @@ namespace milvus::tantivy {
struct RustArrayWrapper {
NO_COPY_OR_ASSIGN(RustArrayWrapper);

explicit RustArrayWrapper(RustArray array) : array_(array) {
explicit RustArrayWrapper(RustArray&& array) {
array_.array = array.array;
array_.len = array.len;
array_.cap = array.cap;
array.array = nullptr;
array.len = 0;
array.cap = 0;
}

RustArrayWrapper(RustArrayWrapper&& other) noexcept {
Expand Down Expand Up @@ -62,4 +69,42 @@ struct RustArrayWrapper {
}
}
};
struct RustResultWrapper {
NO_COPY_OR_ASSIGN(RustResultWrapper);

RustResultWrapper() = default;
explicit RustResultWrapper(RustResult result)
: result_(std::make_unique<RustResult>(result)) {
}

RustResultWrapper(RustResultWrapper&& other) noexcept {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems that RustResultWrapper is not a safe type.
The move construction operation on it make a double free.
I prefer to wrap those ffi type such as RustResult with unique_ptr to make it safe.

result_ = std::move(other.result_);
}

RustResultWrapper&
operator=(RustResultWrapper&& other) noexcept {
if (this != &other) {
free();
result_ = std::move(other.result_);
}

return *this;
}

~RustResultWrapper() {
free();
}

std::unique_ptr<RustResult> result_;

private:
void
free() {
if (result_) {
free_rust_result(*result_);
result_.reset();
}
}
};

} // namespace milvus::tantivy
273 changes: 183 additions & 90 deletions internal/core/thirdparty/tantivy/tantivy-binding/include/tantivy-binding.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,59 @@ struct RustArray {
size_t cap;
};

struct Value {
enum class Tag {
None,
RustArray,
U32,
Ptr,
};

struct None_Body {

};

struct RustArray_Body {
RustArray _0;
};

struct U32_Body {
uint32_t _0;
};

struct Ptr_Body {
void *_0;
};

Tag tag;
union {
None_Body none;
RustArray_Body rust_array;
U32_Body u32;
Ptr_Body ptr;
};
};

struct RustResult {
bool success;
Value value;
const char *error;
};

extern "C" {

void free_rust_array(RustArray array);

void free_rust_result(RustResult result);

void free_rust_error(const char *error);

RustResult test_enum_with_array();

RustResult test_enum_with_ptr();

void free_test_ptr(void *ptr);

void print_vector_of_strings(const char *const *ptr, uintptr_t len);

void *create_hashmap();
Expand All @@ -32,120 +81,164 @@ void hashmap_set_value(void *map, const char *key, const char *value);

void free_hashmap(void *map);

void *tantivy_load_index(const char *path);
RustResult tantivy_load_index(const char *path);

void tantivy_free_index_reader(void *ptr);

void tantivy_reload_index(void *ptr);
RustResult tantivy_reload_index(void *ptr);

uint32_t tantivy_index_count(void *ptr);
RustResult tantivy_index_count(void *ptr);

RustArray tantivy_term_query_i64(void *ptr, int64_t term);
RustResult tantivy_term_query_i64(void *ptr, int64_t term);

RustArray tantivy_lower_bound_range_query_i64(void *ptr, int64_t lower_bound, bool inclusive);
RustResult tantivy_lower_bound_range_query_i64(void *ptr, int64_t lower_bound, bool inclusive);

RustArray tantivy_upper_bound_range_query_i64(void *ptr, int64_t upper_bound, bool inclusive);
RustResult tantivy_upper_bound_range_query_i64(void *ptr, int64_t upper_bound, bool inclusive);

RustArray tantivy_range_query_i64(void *ptr,
int64_t lower_bound,
int64_t upper_bound,
bool lb_inclusive,
bool ub_inclusive);
RustResult tantivy_range_query_i64(void *ptr,
int64_t lower_bound,
int64_t upper_bound,
bool lb_inclusive,
bool ub_inclusive);

RustArray tantivy_term_query_f64(void *ptr, double term);
RustResult tantivy_term_query_f64(void *ptr, double term);

RustArray tantivy_lower_bound_range_query_f64(void *ptr, double lower_bound, bool inclusive);
RustResult tantivy_lower_bound_range_query_f64(void *ptr, double lower_bound, bool inclusive);

RustArray tantivy_upper_bound_range_query_f64(void *ptr, double upper_bound, bool inclusive);
RustResult tantivy_upper_bound_range_query_f64(void *ptr, double upper_bound, bool inclusive);

RustArray tantivy_range_query_f64(void *ptr,
double lower_bound,
double upper_bound,
bool lb_inclusive,
bool ub_inclusive);
RustResult tantivy_range_query_f64(void *ptr,
double lower_bound,
double upper_bound,
bool lb_inclusive,
bool ub_inclusive);

RustArray tantivy_term_query_bool(void *ptr, bool term);
RustResult tantivy_term_query_bool(void *ptr, bool term);

RustArray tantivy_term_query_keyword(void *ptr, const char *term);
RustResult tantivy_term_query_keyword(void *ptr, const char *term);

RustArray tantivy_lower_bound_range_query_keyword(void *ptr,
const char *lower_bound,
bool inclusive);
RustResult tantivy_lower_bound_range_query_keyword(void *ptr,
const char *lower_bound,
bool inclusive);

RustArray tantivy_upper_bound_range_query_keyword(void *ptr,
const char *upper_bound,
bool inclusive);
RustResult tantivy_upper_bound_range_query_keyword(void *ptr,
const char *upper_bound,
bool inclusive);

RustArray tantivy_range_query_keyword(void *ptr,
const char *lower_bound,
const char *upper_bound,
bool lb_inclusive,
bool ub_inclusive);
RustResult tantivy_range_query_keyword(void *ptr,
const char *lower_bound,
const char *upper_bound,
bool lb_inclusive,
bool ub_inclusive);

RustArray tantivy_prefix_query_keyword(void *ptr, const char *prefix);
RustResult tantivy_prefix_query_keyword(void *ptr, const char *prefix);

RustArray tantivy_regex_query(void *ptr, const char *pattern);
RustResult tantivy_regex_query(void *ptr, const char *pattern);

RustArray tantivy_match_query(void *ptr, const char *query);
RustResult tantivy_match_query(void *ptr, const char *query);

void tantivy_register_tokenizer(void *ptr, const char *tokenizer_name, const char *analyzer_params);
RustResult tantivy_register_tokenizer(void *ptr,
const char *tokenizer_name,
const char *analyzer_params);

void *tantivy_create_index(const char *field_name,
TantivyDataType data_type,
const char *path,
uintptr_t num_threads,
uintptr_t overall_memory_budget_in_bytes);
RustResult tantivy_create_index(const char *field_name,
TantivyDataType data_type,
const char *path,
uintptr_t num_threads,
uintptr_t overall_memory_budget_in_bytes);

void tantivy_free_index_writer(void *ptr);

void tantivy_finish_index(void *ptr);

void tantivy_commit_index(void *ptr);

void *tantivy_create_reader_from_writer(void *ptr);

void tantivy_index_add_int8s(void *ptr, const int8_t *array, uintptr_t len, int64_t offset_begin);

void tantivy_index_add_int16s(void *ptr, const int16_t *array, uintptr_t len, int64_t offset_begin);

void tantivy_index_add_int32s(void *ptr, const int32_t *array, uintptr_t len, int64_t offset_begin);

void tantivy_index_add_int64s(void *ptr, const int64_t *array, uintptr_t len, int64_t offset_begin);

void tantivy_index_add_f32s(void *ptr, const float *array, uintptr_t len, int64_t offset_begin);

void tantivy_index_add_f64s(void *ptr, const double *array, uintptr_t len, int64_t offset_begin);

void tantivy_index_add_bools(void *ptr, const bool *array, uintptr_t len, int64_t offset_begin);

void tantivy_index_add_string(void *ptr, const char *s, int64_t offset);

void tantivy_index_add_multi_int8s(void *ptr, const int8_t *array, uintptr_t len, int64_t offset);

void tantivy_index_add_multi_int16s(void *ptr, const int16_t *array, uintptr_t len, int64_t offset);

void tantivy_index_add_multi_int32s(void *ptr, const int32_t *array, uintptr_t len, int64_t offset);

void tantivy_index_add_multi_int64s(void *ptr, const int64_t *array, uintptr_t len, int64_t offset);

void tantivy_index_add_multi_f32s(void *ptr, const float *array, uintptr_t len, int64_t offset);

void tantivy_index_add_multi_f64s(void *ptr, const double *array, uintptr_t len, int64_t offset);

void tantivy_index_add_multi_bools(void *ptr, const bool *array, uintptr_t len, int64_t offset);

void tantivy_index_add_multi_keywords(void *ptr,
const char *const *array,
uintptr_t len,
int64_t offset);

void *tantivy_create_text_writer(const char *field_name,
const char *path,
const char *tokenizer_name,
const char *analyzer_params,
uintptr_t num_threads,
uintptr_t overall_memory_budget_in_bytes,
bool in_ram);
RustResult tantivy_finish_index(void *ptr);

RustResult tantivy_commit_index(void *ptr);

RustResult tantivy_create_reader_from_writer(void *ptr);

RustResult tantivy_index_add_int8s(void *ptr,
const int8_t *array,
uintptr_t len,
int64_t offset_begin);

RustResult tantivy_index_add_int16s(void *ptr,
const int16_t *array,
uintptr_t len,
int64_t offset_begin);

RustResult tantivy_index_add_int32s(void *ptr,
const int32_t *array,
uintptr_t len,
int64_t offset_begin);

RustResult tantivy_index_add_int64s(void *ptr,
const int64_t *array,
uintptr_t len,
int64_t offset_begin);

RustResult tantivy_index_add_f32s(void *ptr,
const float *array,
uintptr_t len,
int64_t offset_begin);

RustResult tantivy_index_add_f64s(void *ptr,
const double *array,
uintptr_t len,
int64_t offset_begin);

RustResult tantivy_index_add_bools(void *ptr,
const bool *array,
uintptr_t len,
int64_t offset_begin);

RustResult tantivy_index_add_string(void *ptr, const char *s, int64_t offset);

RustResult tantivy_index_add_multi_int8s(void *ptr,
const int8_t *array,
uintptr_t len,
int64_t offset);

RustResult tantivy_index_add_multi_int16s(void *ptr,
const int16_t *array,
uintptr_t len,
int64_t offset);

RustResult tantivy_index_add_multi_int32s(void *ptr,
const int32_t *array,
uintptr_t len,
int64_t offset);

RustResult tantivy_index_add_multi_int64s(void *ptr,
const int64_t *array,
uintptr_t len,
int64_t offset);

RustResult tantivy_index_add_multi_f32s(void *ptr,
const float *array,
uintptr_t len,
int64_t offset);

RustResult tantivy_index_add_multi_f64s(void *ptr,
const double *array,
uintptr_t len,
int64_t offset);

RustResult tantivy_index_add_multi_bools(void *ptr,
const bool *array,
uintptr_t len,
int64_t offset);

RustResult tantivy_index_add_multi_keywords(void *ptr,
const char *const *array,
uintptr_t len,
int64_t offset);

RustResult tantivy_create_text_writer(const char *field_name,
const char *path,
const char *tokenizer_name,
const char *analyzer_params,
uintptr_t num_threads,
uintptr_t overall_memory_budget_in_bytes,
bool in_ram);

void free_rust_string(const char *ptr);

Expand All @@ -157,7 +250,7 @@ bool tantivy_token_stream_advance(void *token_stream);

const char *tantivy_token_stream_get_token(void *token_stream);

void *tantivy_create_tokenizer(const char *analyzer_params);
RustResult tantivy_create_tokenizer(const char *analyzer_params);

void *tantivy_clone_tokenizer(void *ptr);

Expand Down
Loading
Loading