Skip to content

Commit

Permalink
enhance: all op(Null) is false in expr (#35527)
Browse files Browse the repository at this point in the history 
#31728

---------

Signed-off-by: lixinguo <[email protected]>
Co-authored-by: lixinguo <[email protected]>
  • Loading branch information
@smellthemoon
smellthemoon and lixinguo authored Oct 17, 2024
1 parent 04c306e commit eb3e458
Show file tree
Hide file tree
Showing 39 changed files with 12,688 additions and 4,209 deletions.
2 changes: 1 addition & 1 deletion internal/core/src/common/FieldData.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ FieldDataImpl<Type, is_type_entire_row>::FillFieldData(
ssize_t byte_count = (element_count + 7) / 8;
// Note: if 'nullable == true` and valid_data is nullptr
// means null_count == 0, will fill it with 0xFF
if (!valid_data) {
if (valid_data == nullptr) {
valid_data_.assign(byte_count, 0xFF);
} else {
std::copy_n(valid_data, byte_count, valid_data_.data());
Expand Down
23 changes: 21 additions & 2 deletions internal/core/src/common/Vector.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
#include <memory>
#include <string>

#include "EasyAssert.h"
#include "Types.h"
#include "common/FieldData.h"

namespace milvus {
Expand Down Expand Up @@ -50,6 +52,7 @@ class BaseVector {
protected:
DataType type_kind_;
size_t length_;
// todo: use null_count to skip some bitset operate
std::optional<size_t> null_count_;
};

Expand All @@ -65,8 +68,8 @@ class ColumnVector final : public BaseVector {
size_t length,
std::optional<size_t> null_count = std::nullopt)
: BaseVector(data_type, length, null_count) {
//todo: support null expr
values_ = InitScalarFieldData(data_type, false, length);
valid_values_ = InitScalarFieldData(data_type, false, length);
}

// ColumnVector(FixedVector<bool>&& data)
Expand All @@ -75,22 +78,37 @@ class ColumnVector final : public BaseVector {
// std::make_shared<FieldData<bool>>(DataType::BOOL, std::move(data));
// }

// // the size is the number of bits
// ColumnVector(TargetBitmap&& bitmap)
// : BaseVector(DataType::INT8, bitmap.size()) {
// values_ = std::make_shared<FieldDataImpl<uint8_t, false>>(
// bitmap.size(), DataType::INT8, false, std::move(bitmap).into());
// }

// the size is the number of bits
ColumnVector(TargetBitmap&& bitmap)
ColumnVector(TargetBitmap&& bitmap, TargetBitmap&& valid_bitmap)
: BaseVector(DataType::INT8, bitmap.size()) {
values_ = std::make_shared<FieldBitsetImpl<uint8_t>>(DataType::INT8,
std::move(bitmap));
valid_values_ = std::make_shared<FieldBitsetImpl<uint8_t>>(
DataType::INT8, std::move(valid_bitmap));
}

virtual ~ColumnVector() override {
values_.reset();
valid_values_.reset();
}

void*
GetRawData() {
return values_->Data();
}

void*
GetValidRawData() {
return valid_values_->Data();
}

template <typename As>
const As*
RawAsValues() const {
Expand All @@ -99,6 +117,7 @@ class ColumnVector final : public BaseVector {

private:
FieldDataPtr values_;
FieldDataPtr valid_values_;
};

using ColumnVectorPtr = std::shared_ptr<ColumnVector>;
Expand Down
7 changes: 5 additions & 2 deletions internal/core/src/exec/expression/AlwaysTrueExpr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,19 @@ PhyAlwaysTrueExpr::Eval(EvalCtx& context, VectorPtr& result) {
? active_count_ - current_pos_
: batch_size_;

// always true no need to skip null
if (real_batch_size == 0) {
result = nullptr;
return;
}

auto res_vec =
std::make_shared<ColumnVector>(TargetBitmap(real_batch_size));
auto res_vec = std::make_shared<ColumnVector>(
TargetBitmap(real_batch_size), TargetBitmap(real_batch_size));
TargetBitmapView res(res_vec->GetRawData(), real_batch_size);
TargetBitmapView valid_res(res_vec->GetValidRawData(), real_batch_size);

res.set();
valid_res.set();

result = res_vec;
current_pos_ += real_batch_size;
Expand Down
129 changes: 116 additions & 13 deletions internal/core/src/exec/expression/BinaryArithOpEvalRangeExpr.cpp
View file

Large diffs are not rendered by default.

72 changes: 35 additions & 37 deletions internal/core/src/exec/expression/BinaryArithOpEvalRangeExpr.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,6 @@ struct ArithOpElementFunc {
}
}
*/

if constexpr (!std::is_same_v<decltype(CmpOpHelper<cmp_op>::op),
void>) {
constexpr auto cmp_op_cvt = CmpOpHelper<cmp_op>::op;
Expand Down Expand Up @@ -282,22 +281,26 @@ struct ArithOpIndexFunc {
HighPrecisonType right_operand) {
TargetBitmap res(size);
for (size_t i = 0; i < size; ++i) {
auto raw = index->Reverse_Lookup(i);
if (!raw.has_value()) {
res[i] = false;
continue;
}
if constexpr (cmp_op == proto::plan::OpType::Equal) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) == val;
res[i] = (raw.value() + right_operand) == val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) == val;
res[i] = (raw.value() - right_operand) == val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) == val;
res[i] = (raw.value() * right_operand) == val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) == val;
res[i] = (raw.value() / right_operand) == val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) == val;
res[i] = (fmod(raw.value(), right_operand)) == val;
} else {
PanicInfo(
OpTypeInvalid,
Expand All @@ -307,20 +310,19 @@ struct ArithOpIndexFunc {
}
} else if constexpr (cmp_op == proto::plan::OpType::NotEqual) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) != val;
res[i] = (raw.value() + right_operand) != val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) != val;
res[i] = (raw.value() - right_operand) != val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) != val;
res[i] = (raw.value() * right_operand) != val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) != val;
res[i] = (raw.value() / right_operand) != val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) != val;
res[i] = (fmod(raw.value(), right_operand)) != val;
} else {
PanicInfo(
OpTypeInvalid,
Expand All @@ -330,20 +332,19 @@ struct ArithOpIndexFunc {
}
} else if constexpr (cmp_op == proto::plan::OpType::GreaterThan) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) > val;
res[i] = (raw.value() + right_operand) > val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) > val;
res[i] = (raw.value() - right_operand) > val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) > val;
res[i] = (raw.value() * right_operand) > val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) > val;
res[i] = (raw.value() / right_operand) > val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) > val;
res[i] = (fmod(raw.value(), right_operand)) > val;
} else {
PanicInfo(
OpTypeInvalid,
Expand All @@ -353,20 +354,19 @@ struct ArithOpIndexFunc {
}
} else if constexpr (cmp_op == proto::plan::OpType::GreaterEqual) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) >= val;
res[i] = (raw.value() + right_operand) >= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) >= val;
res[i] = (raw.value() - right_operand) >= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) >= val;
res[i] = (raw.value() * right_operand) >= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) >= val;
res[i] = (raw.value() / right_operand) >= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) >= val;
res[i] = (fmod(raw.value(), right_operand)) >= val;
} else {
PanicInfo(
OpTypeInvalid,
Expand All @@ -376,20 +376,19 @@ struct ArithOpIndexFunc {
}
} else if constexpr (cmp_op == proto::plan::OpType::LessThan) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) < val;
res[i] = (raw.value() + right_operand) < val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) < val;
res[i] = (raw.value() - right_operand) < val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) < val;
res[i] = (raw.value() * right_operand) < val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) < val;
res[i] = (raw.value() / right_operand) < val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) < val;
res[i] = (fmod(raw.value(), right_operand)) < val;
} else {
PanicInfo(
OpTypeInvalid,
Expand All @@ -399,20 +398,19 @@ struct ArithOpIndexFunc {
}
} else if constexpr (cmp_op == proto::plan::OpType::LessEqual) {
if constexpr (arith_op == proto::plan::ArithOpType::Add) {
res[i] = (index->Reverse_Lookup(i) + right_operand) <= val;
res[i] = (raw.value() + right_operand) <= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Sub) {
res[i] = (index->Reverse_Lookup(i) - right_operand) <= val;
res[i] = (raw.value() - right_operand) <= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mul) {
res[i] = (index->Reverse_Lookup(i) * right_operand) <= val;
res[i] = (raw.value() * right_operand) <= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Div) {
res[i] = (index->Reverse_Lookup(i) / right_operand) <= val;
res[i] = (raw.value() / right_operand) <= val;
} else if constexpr (arith_op ==
proto::plan::ArithOpType::Mod) {
res[i] =
(fmod(index->Reverse_Lookup(i), right_operand)) <= val;
res[i] = (fmod(raw.value(), right_operand)) <= val;
} else {
PanicInfo(
OpTypeInvalid,
Expand Down
Loading

0 comments on commit eb3e458

Please sign in to comment.