Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enhance: all op(Null) is false in expr #35527

Merged
merged 4 commits into from
Oct 17, 2024
Merged

enhance: all op(Null) is false in expr #35527

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a506342
enhance: all op(Null) is false in expr
Aug 16, 2024
d020689
store valid bitset and add test in not expr
Aug 26, 2024
460465c
fix conflict
Sep 13, 2024
37ce45b
fix comment
Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/core/src/common/FieldData.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ FieldDataImpl<Type, is_type_entire_row>::FillFieldData(
ssize_t byte_count = (element_count + 7) / 8;
// Note: if 'nullable == true` and valid_data is nullptr
// means null_count == 0, will fill it with 0xFF
if (!valid_data) {
if (valid_data == nullptr) {
valid_data_.assign(byte_count, 0xFF);
} else {
std::copy_n(valid_data, byte_count, valid_data_.data());
Expand Down
23 changes: 21 additions & 2 deletions internal/core/src/common/Vector.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
#include <memory>
#include <string>

#include "EasyAssert.h"
#include "Types.h"
#include "common/FieldData.h"

namespace milvus {
Expand Down Expand Up @@ -50,6 +52,7 @@ class BaseVector {
protected:
DataType type_kind_;
size_t length_;
// todo: use null_count to skip some bitset operate
std::optional<size_t> null_count_;
};

Expand All @@ -65,8 +68,8 @@ class ColumnVector final : public BaseVector {
size_t length,
std::optional<size_t> null_count = std::nullopt)
: BaseVector(data_type, length, null_count) {
//todo: support null expr
values_ = InitScalarFieldData(data_type, false, length);
valid_values_ = InitScalarFieldData(data_type, false, length);
}

// ColumnVector(FixedVector<bool>&& data)
Expand All @@ -75,22 +78,37 @@ class ColumnVector final : public BaseVector {
// std::make_shared<FieldData<bool>>(DataType::BOOL, std::move(data));
// }

// // the size is the number of bits
// ColumnVector(TargetBitmap&& bitmap)
// : BaseVector(DataType::INT8, bitmap.size()) {
// values_ = std::make_shared<FieldDataImpl<uint8_t, false>>(
// bitmap.size(), DataType::INT8, false, std::move(bitmap).into());
// }

// the size is the number of bits
ColumnVector(TargetBitmap&& bitmap)
ColumnVector(TargetBitmap&& bitmap, TargetBitmap&& valid_bitmap)
: BaseVector(DataType::INT8, bitmap.size()) {
values_ = std::make_shared<FieldBitsetImpl<uint8_t>>(DataType::INT8,
std::move(bitmap));
valid_values_ = std::make_shared<FieldBitsetImpl<uint8_t>>(
DataType::INT8, std::move(valid_bitmap));
}

virtual ~ColumnVector() override {
values_.reset();
valid_values_.reset();
}

void*
GetRawData() {
return values_->Data();
}

void*
GetValidRawData() {
return valid_values_->Data();
}

template <typename As>
const As*
RawAsValues() const {
Expand All @@ -99,6 +117,7 @@ class ColumnVector final : public BaseVector {

private:
FieldDataPtr values_;
FieldDataPtr valid_values_;
};

using ColumnVectorPtr = std::shared_ptr<ColumnVector>;
Expand Down
7 changes: 5 additions & 2 deletions internal/core/src/exec/expression/AlwaysTrueExpr.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,19 @@ PhyAlwaysTrueExpr::Eval(EvalCtx& context, VectorPtr& result) {
? active_count_ - current_pos_
: batch_size_;

// always true no need to skip null
if (real_batch_size == 0) {
result = nullptr;
return;
}

auto res_vec =
std::make_shared<ColumnVector>(TargetBitmap(real_batch_size));
auto res_vec = std::make_shared<ColumnVector>(
TargetBitmap(real_batch_size), TargetBitmap(real_batch_size));
TargetBitmapView res(res_vec->GetRawData(), real_batch_size);
TargetBitmapView valid_res(res_vec->GetValidRawData(), real_batch_size);

res.set();
valid_res.set();

result = res_vec;
current_pos_ += real_batch_size;
Expand Down
129 changes: 116 additions & 13 deletions internal/core/src/exec/expression/BinaryArithOpEvalRangeExpr.cpp
View file
Open in desktop

Large diffs are not rendered by default.

72 changes: 35 additions & 37 deletions internal/core/src/exec/expression/BinaryArithOpEvalRangeExpr.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,6 @@ struct ArithOpElementFunc {
}
}
*/

if constexpr (!std::is_same_v<decltype(CmpOpHelper<cmp_op>::op),
void>) {
constexpr auto cmp_op_cvt = CmpOpHelper<cmp_op>::op;
Expand Down Expand Up @@ -282,22 +281,26 @@ struct ArithOpIndexFunc {
HighPrecisonType right_operand) {
TargetBitmap res(size);
for (size_t i = 0; i < size; ++i) {
auto raw = index->Reverse_Lookup(i);
if (!raw.has_value()) {
res[i] = false;
continue;
}
if constexpr (cmp_op == proto::plan::OpType::Equal) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) == val;
res[i] = (raw.value() + right_operand) == val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) == val;
res[i] = (raw.value() - right_operand) == val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) == val;
res[i] = (raw.value() * right_operand) == val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) == val;
res[i] = (raw.value() / right_operand) == val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) == val;
res[i] = (fmod(raw.value(), right_operand)) == val;
} else {
PanicInfo(
OpTypeInvalid,
Expand All @@ -307,20 +310,19 @@ struct ArithOpIndexFunc {
}
} else if constexpr (cmp_op == proto::plan::OpType::NotEqual) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) != val;
res[i] = (raw.value() + right_operand) != val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) != val;
res[i] = (raw.value() - right_operand) != val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) != val;
res[i] = (raw.value() * right_operand) != val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) != val;
res[i] = (raw.value() / right_operand) != val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) != val;
res[i] = (fmod(raw.value(), right_operand)) != val;
} else {
PanicInfo(
OpTypeInvalid,
Expand All @@ -330,20 +332,19 @@ struct ArithOpIndexFunc {
}
} else if constexpr (cmp_op == proto::plan::OpType::GreaterThan) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) > val;
res[i] = (raw.value() + right_operand) > val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) > val;
res[i] = (raw.value() - right_operand) > val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) > val;
res[i] = (raw.value() * right_operand) > val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) > val;
res[i] = (raw.value() / right_operand) > val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) > val;
res[i] = (fmod(raw.value(), right_operand)) > val;
} else {
PanicInfo(
OpTypeInvalid,
Expand All @@ -353,20 +354,19 @@ struct ArithOpIndexFunc {
}
} else if constexpr (cmp_op == proto::plan::OpType::GreaterEqual) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) >= val;
res[i] = (raw.value() + right_operand) >= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) >= val;
res[i] = (raw.value() - right_operand) >= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) >= val;
res[i] = (raw.value() * right_operand) >= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) >= val;
res[i] = (raw.value() / right_operand) >= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) >= val;
res[i] = (fmod(raw.value(), right_operand)) >= val;
} else {
PanicInfo(
OpTypeInvalid,
Expand All @@ -376,20 +376,19 @@ struct ArithOpIndexFunc {
}
} else if constexpr (cmp_op == proto::plan::OpType::LessThan) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) < val;
res[i] = (raw.value() + right_operand) < val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) < val;
res[i] = (raw.value() - right_operand) < val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) < val;
res[i] = (raw.value() * right_operand) < val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) < val;
res[i] = (raw.value() / right_operand) < val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) < val;
res[i] = (fmod(raw.value(), right_operand)) < val;
} else {
PanicInfo(
OpTypeInvalid,
Expand All @@ -399,20 +398,19 @@ struct ArithOpIndexFunc {
}
} else if constexpr (cmp_op == proto::plan::OpType::LessEqual) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) <= val;
res[i] = (raw.value() + right_operand) <= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) <= val;
res[i] = (raw.value() - right_operand) <= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) <= val;
res[i] = (raw.value() * right_operand) <= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) <= val;
res[i] = (raw.value() / right_operand) <= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) <= val;
res[i] = (fmod(raw.value(), right_operand)) <= val;
} else {
PanicInfo(
OpTypeInvalid,
Expand Down
Loading
Loading