support skip-index based on chunk-metrics to accelerate expr filter(#…

…27925) Signed-off-by: MrPresent-Han <[email protected]>
milvus-io · Nov 14, 2023 · f0e3aff · f0e3aff
1 parent f8aa464
commit f0e3aff
Show file tree

Hide file tree

Showing 13 changed files with 909 additions and 81 deletions.
diff --git a/internal/core/src/index/ScalarIndexSort-inl.h b/internal/core/src/index/ScalarIndexSort-inl.h
@@ -236,6 +236,9 @@ ScalarIndexSort<T>::Range(const T value, const OpType op) {
     TargetBitmap bitset(data_.size());
     auto lb = data_.begin();
     auto ub = data_.end();
+    if (ShouldSkip(value, value, op)) {
+        return bitset;
+    }
     switch (op) {
         case OpType::LessThan:
             ub = std::lower_bound(
@@ -276,6 +279,9 @@ ScalarIndexSort<T>::Range(T lower_bound_value,
          !(lb_inclusive && ub_inclusive))) {
         return bitset;
     }
+    if (ShouldSkip(lower_bound_value, upper_bound_value, OpType::Range)) {
+        return bitset;
+    }
     auto lb = data_.begin();
     auto ub = data_.end();
     if (lb_inclusive) {
@@ -307,4 +313,48 @@ ScalarIndexSort<T>::Reverse_Lookup(size_t idx) const {
     auto offset = idx_to_offsets_[idx];
     return data_[offset].a_;
 }
+
+template <typename T>
+inline bool
+ScalarIndexSort<T>::ShouldSkip(const T lower_value,
+                               const T upper_value,
+                               const milvus::OpType op) {
+    if (!data_.empty()) {
+        auto lower_bound = data_.begin();
+        auto upper_bound = data_.rbegin();
+        bool shouldSkip = false;
+        switch (op) {
+            case OpType::LessThan: {
+                shouldSkip = upper_value <= lower_bound->a_;
+                break;
+            }
+            case OpType::LessEqual: {
+                shouldSkip = upper_value < lower_bound->a_;
+                break;
+            }
+            case OpType::GreaterThan: {
+                shouldSkip = lower_value >= upper_bound->a_;
+                break;
+            }
+            case OpType::GreaterEqual: {
+                shouldSkip = lower_value > upper_bound->a_;
+                break;
+            }
+            case OpType::Range: {
+                shouldSkip = (lower_value > upper_bound->a_) ||
+                             (upper_value < lower_bound->a_);
+                break;
+            }
+            default:
+                throw SegcoreError(
+                    OpTypeInvalid,
+                    fmt::format("Invalid OperatorType for "
+                                "checking scalar index optimization: {}",
+                                op));
+        }
+        return shouldSkip;
+    }
+    return true;
+}
+
 }  // namespace milvus::index
diff --git a/internal/core/src/index/ScalarIndexSort.h b/internal/core/src/index/ScalarIndexSort.h
@@ -82,6 +82,10 @@ class ScalarIndexSort : public ScalarIndex<T> {
     BinarySet
     Upload(const Config& config = {}) override;
 
+ private:
+    bool
+    ShouldSkip(const T lower_value, const T upper_value, const OpType op);
+
  public:
     const std::vector<IndexStructure<T>>&
     GetData() {

diff --git a/internal/core/src/query/generated/ExecExprVisitor.h b/internal/core/src/query/generated/ExecExprVisitor.h
@@ -90,11 +90,15 @@ class ExecExprVisitor : public ExprVisitor {
     }
 
  public:
-    template <typename T, typename IndexFunc, typename ElementFunc>
+    template <typename T,
+              typename IndexFunc,
+              typename ElementFunc,
+              typename SkipIndexFunc>
     auto
     ExecRangeVisitorImpl(FieldId field_id,
                          IndexFunc func,
-                         ElementFunc element_func) -> BitsetType;
+                         ElementFunc element_func,
+                         SkipIndexFunc skip_index_func) -> BitsetType;
 
     template <typename T, typename IndexFunc, typename ElementFunc>
     auto