From 2fa7b01098f79168d11e8b314fe44b254fca072d Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Fri, 11 Aug 2023 10:44:07 -0700 Subject: [PATCH] Add support for `date_nanos` and tests. Signed-off-by: Yury-Fridlyand --- .../org/opensearch/sql/ppl/DataTypeIT.java | 1 + .../opensearch/sql/ppl/SystemFunctionIT.java | 6 ++--- .../opensearch/sql/sql/DateTimeFormatsIT.java | 27 +++++++++++++++++++ .../opensearch/sql/sql/SystemFunctionIT.java | 4 +-- integ-test/src/test/resources/datatypes.json | 2 +- .../datatypes_index_mapping.json | 5 +++- .../date_formats_index_mapping.json | 2 +- .../data/type/OpenSearchDataType.java | 3 +++ .../data/type/OpenSearchDataTypeTest.java | 1 + 9 files changed, 43 insertions(+), 8 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java index 9911c35d8f..53a3b13d0e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java @@ -50,6 +50,7 @@ public void test_nonnumeric_data_types() throws IOException { schema("text_value", "string"), schema("binary_value", "binary"), schema("date_value", "timestamp"), + schema("date_nanos_value", "timestamp"), schema("ip_value", "ip"), schema("object_value", "struct"), schema("nested_value", "array"), diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index de13aa5488..138a039bbf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -59,16 +59,16 @@ public void typeof_opensearch_types() throws IOException { rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE")); response = executeQuery(String.format("source=%s | eval " - + "`text` = typeof(text_value), `date` = typeof(date_value)," + + "`text` = typeof(text_value), `date` = typeof(date_value), `date_nanos` = typeof(date_nanos_value)," + "`boolean` = typeof(boolean_value), `object` = typeof(object_value)," + "`keyword` = typeof(keyword_value), `ip` = typeof(ip_value)," + "`binary` = typeof(binary_value), `geo_point` = typeof(geo_point_value)" // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported //+ ", `nested` = typeof(nested_value)" - + " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`", + + " | fields `text`, `date`, `date_nanos`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`", TEST_INDEX_DATATYPE_NONNUMERIC)); verifyDataRows(response, - rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", + rows("TEXT", "TIMESTAMP", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", "IP", "BINARY", "GEO_POINT")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index fc05e502c5..65c0170148 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -30,6 +31,7 @@ public class DateTimeFormatsIT extends SQLIntegTestCase { public void init() throws Exception { super.init(); loadIndex(Index.DATE_FORMATS); + loadIndex(Index.DATA_TYPE_NONNUMERIC); } @Test @@ -120,6 +122,31 @@ public void testNumericFormats() { rows("1970-01-02 03:55:00", "1970-01-01 00:01:40.5")); } + @Test + @SneakyThrows + public void testDateNanosWithFormats() { + String query = String.format("SELECT hour_minute_second_OR_t_time" + + " FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, + schema("hour_minute_second_OR_t_time", null, "time")); + verifyDataRows(result, + rows("09:07:42"), + rows("09:07:42")); + } + + @Test + @SneakyThrows + public void testDateNanosWithNanos() { + String query = String.format("SELECT date_nanos_value" + + " FROM %s", TEST_INDEX_DATATYPE_NONNUMERIC); + JSONObject result = executeQuery(query); + verifySchema(result, + schema("date_nanos_value", null, "timestamp")); + verifyDataRows(result, + rows("2019-03-24 01:34:46.123456789")); + } + protected JSONObject executeQuery(String query) throws IOException { Request request = new Request("POST", QUERY_API_ENDPOINT); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java index 584cdd05dd..242d222811 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java @@ -47,14 +47,14 @@ public void typeof_opensearch_types() { verifyDataRows(response, rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE")); - response = executeJdbcRequest(String.format("SELECT typeof(text_value)," + response = executeJdbcRequest(String.format("SELECT typeof(text_value), typeof(date_nanos_value)," + "typeof(date_value), typeof(boolean_value), typeof(object_value), typeof(keyword_value)," + "typeof(ip_value), typeof(binary_value), typeof(geo_point_value)" // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported //+ ", typeof(nested_value)" + " from %s;", TEST_INDEX_DATATYPE_NONNUMERIC)); verifyDataRows(response, - rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", + rows("TEXT", "TIMESTAMP", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", "IP", "BINARY", "GEO_POINT")); } } diff --git a/integ-test/src/test/resources/datatypes.json b/integ-test/src/test/resources/datatypes.json index ea3290ee64..70ddd28763 100644 --- a/integ-test/src/test/resources/datatypes.json +++ b/integ-test/src/test/resources/datatypes.json @@ -1,2 +1,2 @@ {"index":{"_id":"1"}} -{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }} +{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }} diff --git a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json index 8c1759b369..5908114201 100644 --- a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json @@ -13,10 +13,13 @@ "binary_value": { "type": "binary" }, - "date_value": { + "date_value": { "type" : "date", "format": "yyyy-MM-dd HH:mm:ss" }, + "date_nanos_value": { + "type" : "date_nanos" + }, "ip_value": { "type": "ip" }, diff --git a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json index 65811f8d9e..0b6daaacb4 100644 --- a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json @@ -298,7 +298,7 @@ "format": "yyyy-MM-dd||epoch_millis" }, "hour_minute_second_OR_t_time" : { - "type" : "date", + "type" : "date_nanos", "format": "hour_minute_second||t_time" }, "custom_timestamp" : { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 273b980d2a..01585600d0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -35,6 +35,7 @@ public enum MappingType { GeoPoint("geo_point", ExprCoreType.UNKNOWN), Binary("binary", ExprCoreType.UNKNOWN), Date("date", ExprCoreType.TIMESTAMP), + DateNanos("date_nanos", ExprCoreType.TIMESTAMP), Object("object", ExprCoreType.STRUCT), Nested("nested", ExprCoreType.ARRAY), Byte("byte", ExprCoreType.BYTE), @@ -137,6 +138,7 @@ public static Map parseMapping(Map i * @param mappingType A mapping type. * @return An instance or inheritor of `OpenSearchDataType`. */ + @SuppressWarnings("unchecked") public static OpenSearchDataType of(MappingType mappingType, Map innerMap) { OpenSearchDataType res = instances.getOrDefault(mappingType.toString(), new OpenSearchDataType(mappingType) @@ -162,6 +164,7 @@ public static OpenSearchDataType of(MappingType mappingType, Map case Binary: return OpenSearchBinaryType.of(); case Ip: return OpenSearchIpType.of(); case Date: + case DateNanos: // Default date formatter is used when "" is passed as the second parameter String format = (String) innerMap.getOrDefault("format", ""); return OpenSearchDateType.of(format); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 8d69b3d855..c74d06d262 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -106,6 +106,7 @@ private static Stream getTestDataWithType() { Arguments.of(MappingType.Double, "double", DOUBLE), Arguments.of(MappingType.Boolean, "boolean", BOOLEAN), Arguments.of(MappingType.Date, "date", TIMESTAMP), + Arguments.of(MappingType.DateNanos, "date", TIMESTAMP), Arguments.of(MappingType.Object, "object", STRUCT), Arguments.of(MappingType.Nested, "nested", ARRAY), Arguments.of(MappingType.GeoPoint, "geo_point",