Skip to content

Commit

Permalink
fix conflict
Browse files Browse the repository at this point in the history 
Signed-off-by: lixinguo <[email protected]>
  • Loading branch information
lixinguo committed Sep 23, 2024
1 parent 0d59c8b commit 6c93c57
Show file tree
Hide file tree
Showing 11 changed files with 320 additions and 183 deletions.
7 changes: 4 additions & 3 deletions internal/core/src/exec/expression/Expr.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ class SegmentExpr : public Expr {
template <typename T>
TargetBitmap
ProcessDataChunksForValid() {
TargetBitmap valid_result;
TargetBitmap valid_result(batch_size_);
valid_result.set();
int64_t processed_size = 0;
for (size_t i = current_data_chunk_; i < num_data_chunk_; i++) {
Expand All @@ -385,7 +385,7 @@ class SegmentExpr : public Expr {
valid_data += data_pos;
for (int i = 0; i < size; i++) {
if (!valid_data[i]) {
valid_result[i] = false;
valid_result[i + data_pos] = false;

Check warning on line 388 in internal/core/src/exec/expression/Expr.h

View check run for this annotation

Codecov / codecov/patch

internal/core/src/exec/expression/Expr.h#L385-L388 

Added lines #L385 - L388 were not covered by tests
}
}
processed_size += size;
Expand Down Expand Up @@ -466,12 +466,13 @@ class SegmentExpr : public Expr {
auto res = std::move(func(index, values...));
auto valid_res = index->IsNotNull();
cached_match_res_ = std::make_shared<TargetBitmap>(std::move(res));
cached_index_chunk_valid_res_ = std::move(valid_result);
cached_index_chunk_valid_res_ = std::move(valid_res);
if (cached_match_res_->size() < active_count_) {
// some entities are not visible in inverted index.
// only happend on growing segment.
TargetBitmap tail(active_count_ - cached_match_res_->size());
cached_match_res_->append(tail);
cached_index_chunk_valid_res_.append(tail);

Check warning on line 475 in internal/core/src/exec/expression/Expr.h

View check run for this annotation

Codecov / codecov/patch

internal/core/src/exec/expression/Expr.h#L475 

Added line #L475 was not covered by tests
}
}

Expand Down
4 changes: 4 additions & 0 deletions internal/core/src/exec/expression/LogicalBinaryExpr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ PhyLogicalBinaryExpr::Eval(EvalCtx& context, VectorPtr& result) {
"unsupported logical operator: {}",
expr_->GetOpTypeString());
}
TargetBitmapView lvalid_view(lflat->GetValidRawData(), size);
TargetBitmapView rvalid_view(rflat->GetValidRawData(), size);
LogicalElementFunc<LogicalOpType::Or> func;
func(lvalid_view, rvalid_view, size);

Check warning on line 51 in internal/core/src/exec/expression/LogicalBinaryExpr.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/exec/expression/LogicalBinaryExpr.cpp#L48-L51 

Added lines #L48 - L51 were not covered by tests
result = std::move(left);
}

Expand Down
7 changes: 6 additions & 1 deletion internal/core/src/exec/operator/FilterBitsNode.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ PhyFilterBitsNode::GetOutput() {
operator_context_->get_exec_context(), exprs_.get(), input_.get());

TargetBitmap bitset;
TargetBitmap valid_bitset;
while (num_processed_rows_ < need_process_rows_) {
exprs_->Eval(0, 1, true, eval_ctx, results_);

Expand All @@ -79,13 +80,17 @@ PhyFilterBitsNode::GetOutput() {
auto col_vec_size = col_vec->size();
TargetBitmapView view(col_vec->GetRawData(), col_vec_size);
bitset.append(view);
TargetBitmapView valid_view(col_vec->GetValidRawData(), col_vec_size);
valid_bitset.append(valid_view);
num_processed_rows_ += col_vec_size;
}
bitset.flip();
Assert(bitset.size() == need_process_rows_);
Assert(valid_bitset.size() == need_process_rows_);
// num_processed_rows_ = need_process_rows_;
std::vector<VectorPtr> col_res;
col_res.push_back(std::make_shared<ColumnVector>(std::move(bitset)));
col_res.push_back(std::make_shared<ColumnVector>(std::move(bitset),
std::move(valid_bitset)));
std::chrono::high_resolution_clock::time_point scalar_end =
std::chrono::high_resolution_clock::now();
double scalar_cost =
Expand Down
9 changes: 5 additions & 4 deletions internal/core/src/exec/operator/MvccNode.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,13 @@ PhyMvccNode::GetOutput() {
return nullptr;
}

auto col_input =
is_source_node_
? std::make_shared<ColumnVector>(TargetBitmap(active_count_))
: GetColumnVector(input_);
auto col_input = is_source_node_ ? std::make_shared<ColumnVector>(
TargetBitmap(active_count_),

Check warning on line 56 in internal/core/src/exec/operator/MvccNode.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/exec/operator/MvccNode.cpp#L56 

Added line #L56 was not covered by tests
TargetBitmap(active_count_))
: GetColumnVector(input_);

TargetBitmapView data(col_input->GetRawData(), col_input->size());
// need to expose null?
segment_->mask_with_timestamps(data, query_timestamp_);
segment_->mask_with_delete(data, active_count_, query_timestamp_);
is_finished_ = true;
Expand Down
13 changes: 9 additions & 4 deletions internal/core/src/index/StringIndexMarisa.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ StringIndexMarisa::Build(size_t n,
}

trie_.build(keyset, MARISA_LABEL_ORDER);
fill_str_ids(n, values);
fill_str_ids(n, values, valid_data);
fill_offsets();

built_ = true;
Expand Down Expand Up @@ -218,7 +218,7 @@ StringIndexMarisa::LoadWithoutAssemble(const BinarySet& set,

auto str_ids = set.GetByName(MARISA_STR_IDS);
auto str_ids_len = str_ids->size;
str_ids_.resize(str_ids_len / sizeof(size_t));
str_ids_.resize(str_ids_len / sizeof(size_t), MARISA_NULL_KEY_ID);
memcpy(str_ids_.data(), str_ids->data.get(), str_ids_len);

fill_offsets();
Expand Down Expand Up @@ -496,9 +496,14 @@ StringIndexMarisa::PrefixMatch(std::string_view prefix) {
}

void
StringIndexMarisa::fill_str_ids(size_t n, const std::string* values) {
str_ids_.resize(n);
StringIndexMarisa::fill_str_ids(size_t n,
const std::string* values,
const bool* valid_data) {
str_ids_.resize(n, MARISA_NULL_KEY_ID);
for (size_t i = 0; i < n; i++) {
if (valid_data && !valid_data[i]) {
continue;
}
auto str = values[i];
auto str_id = lookup(str);
AssertInfo(valid_str_id(str_id), "invalid marisa key");
Expand Down
4 changes: 2 additions & 2 deletions internal/core/src/index/StringIndexMarisa.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ class StringIndexMarisa : public StringIndex {

private:
void
fill_str_ids(size_t n, const std::string* values);
fill_str_ids(size_t n, const std::string* values, const bool* valid_data);

void
fill_offsets();
Expand All @@ -124,7 +124,7 @@ class StringIndexMarisa : public StringIndex {
private:
Config config_;
marisa::Trie trie_;
std::vector<size_t> str_ids_; // used to retrieve.
std::vector<int64_t> str_ids_; // used to retrieve.
std::map<size_t, std::vector<size_t>> str_ids_to_offsets_;
bool built_ = false;
std::shared_ptr<storage::MemFileManagerImpl> file_manager_;
Expand Down
4 changes: 3 additions & 1 deletion internal/core/src/segcore/InsertRecord.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,9 @@ class ThreadSafeValidData {
data_.resize(length_ + num_rows);
}
auto src = data->valid_data().data();
std::copy_n(src, num_rows, data_.data() + length_);
for (size_t i = 0; i < num_rows; i++) {
data_[length_ + i] = src[i];
}
length_ += num_rows;
}
}
Expand Down
50 changes: 50 additions & 0 deletions internal/core/src/segcore/Utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -681,6 +681,11 @@ ReverseDataFromIndex(const index::IndexBase* index,
data_array->set_field_id(field_meta.get_id().get());
data_array->set_type(static_cast<milvus::proto::schema::DataType>(
field_meta.get_data_type()));
auto nullable = field_meta.is_nullable();
std::vector<bool> valid_data;
if (nullable) {
valid_data.resize(count);

Check warning on line 687 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L687 

Added line #L687 was not covered by tests
}

auto scalar_array = data_array->mutable_scalars();
switch (data_type) {
Expand All @@ -690,9 +695,14 @@ ReverseDataFromIndex(const index::IndexBase* index,
std::vector<bool> raw_data(count);
for (int64_t i = 0; i < count; ++i) {
auto value = ptr->Reverse_Lookup(seg_offsets[i]);

Check warning on line 697 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L697 

Added line #L697 was not covered by tests
// if has no value, means nullable must be true, no need to check nullable again here
if (!value.has_value()) {
valid_data[i] = false;
continue;

Check warning on line 701 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L699-L701 

Added lines #L699 - L701 were not covered by tests
}
if (nullable) {
valid_data[i] = true;

Check warning on line 704 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L703-L704 

Added lines #L703 - L704 were not covered by tests
}
raw_data[i] = ptr->Reverse_Lookup(seg_offsets[i]).value();

Check warning on line 706 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L706 

Added line #L706 was not covered by tests
}
auto obj = scalar_array->mutable_bool_data();
Expand All @@ -705,9 +715,14 @@ ReverseDataFromIndex(const index::IndexBase* index,
std::vector<int8_t> raw_data(count);
for (int64_t i = 0; i < count; ++i) {
auto value = ptr->Reverse_Lookup(seg_offsets[i]);
// if has no value, means nullable must be true, no need to check nullable again here
if (!value.has_value()) {
valid_data[i] = false;
continue;

Check warning on line 721 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L720-L721 

Added lines #L720 - L721 were not covered by tests
}
if (nullable) {
valid_data[i] = true;

Check warning on line 724 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L724 

Added line #L724 was not covered by tests
}
raw_data[i] = ptr->Reverse_Lookup(seg_offsets[i]).value();
}
auto obj = scalar_array->mutable_int_data();
Expand All @@ -720,9 +735,14 @@ ReverseDataFromIndex(const index::IndexBase* index,
std::vector<int16_t> raw_data(count);
for (int64_t i = 0; i < count; ++i) {
auto value = ptr->Reverse_Lookup(seg_offsets[i]);
// if has no value, means nullable must be true, no need to check nullable again here
if (!value.has_value()) {
valid_data[i] = false;
continue;

Check warning on line 741 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L740-L741 

Added lines #L740 - L741 were not covered by tests
}
if (nullable) {
valid_data[i] = true;

Check warning on line 744 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L744 

Added line #L744 was not covered by tests
}
raw_data[i] = ptr->Reverse_Lookup(seg_offsets[i]).value();
}
auto obj = scalar_array->mutable_int_data();
Expand All @@ -735,9 +755,14 @@ ReverseDataFromIndex(const index::IndexBase* index,
std::vector<int32_t> raw_data(count);
for (int64_t i = 0; i < count; ++i) {
auto value = ptr->Reverse_Lookup(seg_offsets[i]);
// if has no value, means nullable must be true, no need to check nullable again here
if (!value.has_value()) {
valid_data[i] = false;
continue;

Check warning on line 761 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L760-L761 

Added lines #L760 - L761 were not covered by tests
}
if (nullable) {
valid_data[i] = true;

Check warning on line 764 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L764 

Added line #L764 was not covered by tests
}
raw_data[i] = ptr->Reverse_Lookup(seg_offsets[i]).value();
}
auto obj = scalar_array->mutable_int_data();
Expand All @@ -750,9 +775,14 @@ ReverseDataFromIndex(const index::IndexBase* index,
std::vector<int64_t> raw_data(count);
for (int64_t i = 0; i < count; ++i) {
auto value = ptr->Reverse_Lookup(seg_offsets[i]);
// if has no value, means nullable must be true, no need to check nullable again here
if (!value.has_value()) {
valid_data[i] = false;
continue;

Check warning on line 781 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L780-L781 

Added lines #L780 - L781 were not covered by tests
}
if (nullable) {
valid_data[i] = true;

Check warning on line 784 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L784 

Added line #L784 was not covered by tests
}
raw_data[i] = ptr->Reverse_Lookup(seg_offsets[i]).value();
}
auto obj = scalar_array->mutable_long_data();
Expand All @@ -765,9 +795,14 @@ ReverseDataFromIndex(const index::IndexBase* index,
std::vector<float> raw_data(count);
for (int64_t i = 0; i < count; ++i) {
auto value = ptr->Reverse_Lookup(seg_offsets[i]);
// if has no value, means nullable must be true, no need to check nullable again here
if (!value.has_value()) {
valid_data[i] = false;
continue;

Check warning on line 801 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L800-L801 

Added lines #L800 - L801 were not covered by tests
}
if (nullable) {
valid_data[i] = true;

Check warning on line 804 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L804 

Added line #L804 was not covered by tests
}
raw_data[i] = ptr->Reverse_Lookup(seg_offsets[i]).value();
}
auto obj = scalar_array->mutable_float_data();
Expand All @@ -780,9 +815,14 @@ ReverseDataFromIndex(const index::IndexBase* index,
std::vector<double> raw_data(count);
for (int64_t i = 0; i < count; ++i) {
auto value = ptr->Reverse_Lookup(seg_offsets[i]);
// if has no value, means nullable must be true, no need to check nullable again here
if (!value.has_value()) {
valid_data[i] = false;
continue;

Check warning on line 821 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L820-L821 

Added lines #L820 - L821 were not covered by tests
}
if (nullable) {
valid_data[i] = true;

Check warning on line 824 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L824 

Added line #L824 was not covered by tests
}
raw_data[i] = ptr->Reverse_Lookup(seg_offsets[i]).value();
}
auto obj = scalar_array->mutable_double_data();
Expand All @@ -795,9 +835,14 @@ ReverseDataFromIndex(const index::IndexBase* index,
std::vector<std::string> raw_data(count);
for (int64_t i = 0; i < count; ++i) {
auto value = ptr->Reverse_Lookup(seg_offsets[i]);

Check warning on line 837 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L837 

Added line #L837 was not covered by tests
// if has no value, means nullable must be true, no need to check nullable again here
if (!value.has_value()) {
valid_data[i] = false;
continue;

Check warning on line 841 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L839-L841 

Added lines #L839 - L841 were not covered by tests
}
if (nullable) {
valid_data[i] = true;

Check warning on line 844 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L843-L844 

Added lines #L843 - L844 were not covered by tests
}
raw_data[i] = ptr->Reverse_Lookup(seg_offsets[i]).value();

Check warning on line 846 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L846 

Added line #L846 was not covered by tests
}
auto obj = scalar_array->mutable_string_data();
Expand All @@ -810,6 +855,11 @@ ReverseDataFromIndex(const index::IndexBase* index,
}
}

if (nullable) {
*(data_array->mutable_valid_data()) = {valid_data.begin(),
valid_data.end()};

Check warning on line 860 in internal/core/src/segcore/Utils.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/segcore/Utils.cpp#L859-L860 

Added lines #L859 - L860 were not covered by tests
}

return data_array;
}

Expand Down
Loading

0 comments on commit 6c93c57

Please sign in to comment.