fix:fix term expr overflow bug (#36525)

#36520 Signed-off-by: luzhang <[email protected]> Co-authored-by: luzhang <[email protected]>
milvus-io · Sep 26, 2024 · 0799d92 · 0799d92
1 parent b60164b
commit 0799d92
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 7 deletions.
diff --git a/internal/core/src/exec/expression/TermExpr.cpp b/internal/core/src/exec/expression/TermExpr.cpp
@@ -481,14 +481,12 @@ PhyTermFilterExpr::ExecVisitorImplForIndex() {
 
     std::vector<IndexInnerType> vals;
     for (auto& val : expr_->vals_) {
-        auto converted_val = GetValueFromProto<T>(val);
         // Integral overflow process
-        if constexpr (std::is_integral_v<T>) {
-            if (milvus::query::out_of_range<T>(converted_val)) {
-                continue;
-            }
+        bool overflowed = false;
+        auto converted_val = GetValueFromProtoWithOverflow<T>(val, overflowed);
+        if (!overflowed) {
+            vals.emplace_back(converted_val);
         }
-        vals.emplace_back(converted_val);
     }
     auto execute_sub_batch = [](Index* index_ptr,
                                 const std::vector<IndexInnerType>& vals) {

diff --git a/tests/python_client/testcases/test_mix_scenes.py b/tests/python_client/testcases/test_mix_scenes.py
@@ -1,6 +1,7 @@
 import re
 import math  # do not remove `math`
 import pytest
+import random
 import numpy as np
 from pymilvus import DataType, AnnSearchRequest, RRFRanker, WeightedRanker
 
@@ -455,6 +456,37 @@ def setup_class(self):
         # prepare data (> 1024 triggering index building)
         self.insert_data = cf.gen_field_values(self.collection_wrap.schema, nb=self.nb)
 
+    @pytest.mark.tags(CaseLabel.L2)
+    @pytest.mark.parametrize("range_num, counts", [
+        ([-100, 200], 10),
+        ([2000, 5000], 10),
+        ([3000, 4000], 5),
+        ([0, 0], 1)])
+    @pytest.mark.parametrize("expr_field", ['INT8', 'INT16', 'INT32', 'INT64'])
+    @pytest.mark.parametrize("limit", [1, 10, 3000])
+    def test_bitmap_index_query_with_int_in(self, range_num, counts, expr_field, limit):
+        """
+        target:
+            1. check number operation
+        method:
+            1. prepare some data and  build `BITMAP index` on scalar fields
+            2. query with the different expr(in) and limit
+            3. check query result
+        expected:
+            1. query response equal to min(insert data, limit)
+        """
+        # random set expr list
+        range_numbers = [random.randint(*range_num) for _ in range(counts)]
+
+        # the total number of inserted data that matches the expression
+        expr_data = [i for i in self.insert_data.get(expr_field, []) if i in range_numbers]
+        expr_count = len(expr_data)
+
+        # query
+        res, _ = self.collection_wrap.query(expr=f"{expr_field} in {range_numbers}", limit=limit,
+                                        output_fields=[expr_field])
+        assert len(res) == min(expr_count, limit), f"actual: {len(res)} == expect: {min(expr_count, limit)}, {expr_data}"
+
     @pytest.fixture(scope="class", autouse=True)
     def prepare_data(self):
         self.collection_wrap.insert(data=list(self.insert_data.values()), check_task=CheckTasks.check_insert_result)
@@ -1615,4 +1647,4 @@ def test_search_pagination_group_size(self):
         grpby_field_values = []
         for i in range(limit * page_rounds):
             grpby_field_values.append(total_res[0][i].fields.get(grpby_field))
-        assert len(grpby_field_values) == len(set(grpby_field_values))
+        assert len(grpby_field_values) == len(set(grpby_field_values))