Updating Ip fields to use doc_values to search (#11508) (#12078)

* Updating Ip fields to use doc_values to search * Fix IP tests * Fix skip to allow yaml test to pass on main * Update tests to use existing test file * Changing skip version to match bwc * Using exact match instead of range * Spotless * Fix IP field tests * Fix spotless + precommit failure * Get point out of query and into value * Fix term tests * Add skip test logic to only doc_values test --------- (cherry picked from commit cc9ee9d) Signed-off-by: Harsha Vamsi Kalluri <[email protected]> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Signed-off-by: Andriy Redko <[email protected]> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Jan 31, 2024 · de636c1 · de636c1
1 parent 7e3c080
commit de636c1
Show file tree

Hide file tree

Showing 4 changed files with 323 additions and 48 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -127,6 +127,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Ensure Jackson default maximums introduced in 2.16.0 do not conflict with OpenSearch settings ([#11890](https://github.com/opensearch-project/OpenSearch/pull/11890))
 - Extract cluster management for integration tests into JUnit test rule out of OpenSearchIntegTestCase ([#11877](https://github.com/opensearch-project/OpenSearch/pull/11877)), ([#12000](https://github.com/opensearch-project/OpenSearch/pull/12000))
 - Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2 ([#11968](https://github.com/opensearch-project/OpenSearch/pull/11968))
+- Updates IpField to be searchable when only `doc_values` are enabled ([#11508](https://github.com/opensearch-project/OpenSearch/pull/11508))
 
 ### Deprecated
 

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml
@@ -47,18 +47,22 @@ setup:
                 type: unsigned_long
                 index: true
                 doc_values: true
+              ip_field:
+                type: ip
+                index: true
+                doc_values: true
 
   - do:
       bulk:
         index: test-iodvq
         refresh: true
         body:
           - '{"index": {"_index": "test-iodvq", "_id": "1" }}'
-          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800 }'
+          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
           - '{ "index": { "_index": "test-iodvq", "_id": "2" }}'
-          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801 }'
+          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
           - '{ "index": { "_index": "test-iodvq", "_id": "3" } }'
-          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802 }'
+          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'
 
   - do:
       search:
@@ -162,7 +166,6 @@ setup:
 
   - match: { hits.total: 1 }
 
-
   - do:
       search:
         rest_total_hits_as_int: true
@@ -174,6 +177,16 @@ setup:
 
   - match: { hits.total: 1 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            term:
+              ip_field: "192.168.0.1"
+
+  - match: {hits.total: 1}
 
   - do:
       search:
@@ -186,7 +199,6 @@ setup:
 
   - match: { hits.total: 2 }
 
-
   - do:
       search:
         rest_total_hits_as_int: true
@@ -264,6 +276,17 @@ setup:
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            terms:
+              ip_field: ["192.168.0.1", "192.168.0.2"]
+
+  - match: { hits.total: 2 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -384,6 +407,19 @@ setup:
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range:
+              ip_field:
+                gte: "192.168.0.1"
+                lte: "192.168.0.2"
+
+  - match: { hits.total: 2 }
+
 ---
 "search on fields with only index enabled":
   - do:
@@ -428,18 +464,22 @@ setup:
                 type: unsigned_long
                 index: true
                 doc_values: false
+              ip_field:
+                type: ip
+                index: true
+                doc_values: false
 
   - do:
       bulk:
         index: test-index
         refresh: true
         body:
           - '{"index": {"_index": "test-index", "_id": "1" }}'
-          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800 }'
+          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
           - '{ "index": { "_index": "test-index", "_id": "2" }}'
-          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801 }'
+          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
           - '{ "index": { "_index": "test-index", "_id": "3" } }'
-          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802 }'
+          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'
 
   - do:
       search:
@@ -465,7 +505,6 @@ setup:
 
   - match: { hits.total: 2 }
 
-
   - do:
       search:
         rest_total_hits_as_int: true
@@ -555,6 +594,16 @@ setup:
 
   - match: { hits.total: 1 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            term:
+              ip_field: "192.168.0.1"
+
+  - match: {hits.total: 1}
 
   - do:
       search:
@@ -567,7 +616,6 @@ setup:
 
   - match: { hits.total: 2 }
 
-
   - do:
       search:
         rest_total_hits_as_int: true
@@ -645,6 +693,17 @@ setup:
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            terms:
+              ip_field: ["192.168.0.1", "192.168.0.2"]
+
+  - match: { hits.total: 2 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -765,8 +824,24 @@ setup:
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range:
+              ip_field:
+                gte: "192.168.0.1"
+                lte: "192.168.0.2"
+
+  - match: { hits.total: 2 }
 ---
 "search on fields with only doc_values enabled":
+  - skip:
+      features: [ "headers" ]
+      version: " - 2.99.99"
+      reason: "searching with only doc_values was added in 3.0.0"
   - do:
       indices.create:
         index: test-doc-values
@@ -809,18 +884,22 @@ setup:
                 type: unsigned_long
                 index: false
                 doc_values: true
+              ip_field:
+                type: ip
+                index: false
+                doc_values: true
 
   - do:
       bulk:
         index: test-doc-values
         refresh: true
         body:
           - '{"index": {"_index": "test-doc-values", "_id": "1" }}'
-          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800 }'
+          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
           - '{ "index": { "_index": "test-doc-values", "_id": "2" }}'
-          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801 }'
+          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
           - '{ "index": { "_index": "test-doc-values", "_id": "3" } }'
-          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802 }'
+          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'
 
   - do:
       search:
@@ -846,7 +925,6 @@ setup:
 
   - match: { hits.total: 2 }
 
-
   - do:
       search:
         rest_total_hits_as_int: true
@@ -924,7 +1002,6 @@ setup:
 
   - match: { hits.total: 1 }
 
-
   - do:
       search:
         rest_total_hits_as_int: true
@@ -936,6 +1013,16 @@ setup:
 
   - match: { hits.total: 1 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            term:
+              ip_field: "192.168.0.3"
+
+  - match: { hits.total: 1 }
 
   - do:
       search:
@@ -948,7 +1035,6 @@ setup:
 
   - match: { hits.total: 2 }
 
-
   - do:
       search:
         rest_total_hits_as_int: true
@@ -1145,3 +1231,16 @@ setup:
             }
 
   - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range:
+              ip_field:
+                gte: "192.168.0.1"
+                lte: "192.168.0.2"
+
+  - match: { hits.total: 2 }
diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java
@@ -36,7 +36,9 @@
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
@@ -222,25 +224,48 @@ protected Object parseSourceValue(Object value) {
 
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
-            failIfNotIndexed();
+            failIfNotIndexedAndNoDocValues();
+            Query query;
             if (value instanceof InetAddress) {
-                return InetAddressPoint.newExactQuery(name(), (InetAddress) value);
+                query = InetAddressPoint.newExactQuery(name(), (InetAddress) value);
             } else {
                 if (value instanceof BytesRef) {
                     value = ((BytesRef) value).utf8ToString();
                 }
+                String term = value.toString();
+                if (term.contains("/")) {
+                    final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
+                    query = InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
+                } else {
+                    InetAddress address = InetAddresses.forString(term);
+                    query = InetAddressPoint.newExactQuery(name(), address);
+                }
+            }
+            if (isSearchable() && hasDocValues()) {
+                String term = value.toString();
+                if (term.contains("/")) {
+                    final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
+                    return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
+                }
+                return new IndexOrDocValuesQuery(
+                    query,
+                    SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint()))
+                );
+            }
+            if (hasDocValues()) {
                 String term = value.toString();
                 if (term.contains("/")) {
                     final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
                     return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
                 }
-                InetAddress address = InetAddresses.forString(term);
-                return InetAddressPoint.newExactQuery(name(), address);
+                return SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint()));
             }
+            return query;
         }
 
         @Override
         public Query termsQuery(List<?> values, QueryShardContext context) {
+            failIfNotIndexedAndNoDocValues();
             InetAddress[] addresses = new InetAddress[values.size()];
             int i = 0;
             for (Object value : values) {
@@ -265,14 +290,32 @@ public Query termsQuery(List<?> values, QueryShardContext context) {
 
         @Override
         public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
-            failIfNotIndexed();
-            return rangeQuery(
-                lowerTerm,
-                upperTerm,
-                includeLower,
-                includeUpper,
-                (lower, upper) -> InetAddressPoint.newRangeQuery(name(), lower, upper)
-            );
+            failIfNotIndexedAndNoDocValues();
+            return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (lower, upper) -> {
+                Query query = InetAddressPoint.newRangeQuery(name(), lower, upper);
+                if (isSearchable() && hasDocValues()) {
+                    return new IndexOrDocValuesQuery(
+                        query,
+                        SortedSetDocValuesField.newSlowRangeQuery(
+                            ((PointRangeQuery) query).getField(),
+                            new BytesRef(((PointRangeQuery) query).getLowerPoint()),
+                            new BytesRef(((PointRangeQuery) query).getUpperPoint()),
+                            true,
+                            true
+                        )
+                    );
+                }
+                if (hasDocValues()) {
+                    return SortedSetDocValuesField.newSlowRangeQuery(
+                        ((PointRangeQuery) query).getField(),
+                        new BytesRef(((PointRangeQuery) query).getLowerPoint()),
+                        new BytesRef(((PointRangeQuery) query).getUpperPoint()),
+                        true,
+                        true
+                    );
+                }
+                return query;
+            });
         }
 
         /**