Add support for date_nanos and tests.

Signed-off-by: Yury-Fridlyand <[email protected]>
Bit-Quill · Aug 16, 2023 · 2fa7b01 · 2fa7b01
1 parent 5381a6f
commit 2fa7b01
Show file tree

Hide file tree

Showing 9 changed files with 43 additions and 8 deletions.
diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java
@@ -50,6 +50,7 @@ public void test_nonnumeric_data_types() throws IOException {
         schema("text_value", "string"),
         schema("binary_value", "binary"),
         schema("date_value", "timestamp"),
+        schema("date_nanos_value", "timestamp"),
         schema("ip_value", "ip"),
         schema("object_value", "struct"),
         schema("nested_value", "array"),

diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java
@@ -59,16 +59,16 @@ public void typeof_opensearch_types() throws IOException {
         rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE"));
 
     response = executeQuery(String.format("source=%s | eval "
-        + "`text` = typeof(text_value), `date` = typeof(date_value),"
+        + "`text` = typeof(text_value), `date` = typeof(date_value), `date_nanos` = typeof(date_nanos_value),"
         + "`boolean` = typeof(boolean_value), `object` = typeof(object_value),"
         + "`keyword` = typeof(keyword_value), `ip` = typeof(ip_value),"
         + "`binary` = typeof(binary_value), `geo_point` = typeof(geo_point_value)"
         // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported
         //+ ", `nested` = typeof(nested_value)"
-        + " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`",
+        + " | fields `text`, `date`, `date_nanos`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`",
             TEST_INDEX_DATATYPE_NONNUMERIC));
     verifyDataRows(response,
-        rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
+        rows("TEXT", "TIMESTAMP", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
                 "IP", "BINARY", "GEO_POINT"));
   }
 }
diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java
@@ -6,6 +6,7 @@
 
 package org.opensearch.sql.sql;
 
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS;
 import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT;
 import static org.opensearch.sql.util.MatcherUtils.rows;
@@ -30,6 +31,7 @@ public class DateTimeFormatsIT extends SQLIntegTestCase {
   public void init() throws Exception {
     super.init();
     loadIndex(Index.DATE_FORMATS);
+    loadIndex(Index.DATA_TYPE_NONNUMERIC);
   }
 
   @Test
@@ -120,6 +122,31 @@ public void testNumericFormats() {
         rows("1970-01-02 03:55:00", "1970-01-01 00:01:40.5"));
   }
 
+  @Test
+  @SneakyThrows
+  public void testDateNanosWithFormats() {
+    String query = String.format("SELECT hour_minute_second_OR_t_time"
+        + " FROM %s", TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result,
+        schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result,
+        rows("09:07:42"),
+        rows("09:07:42"));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosWithNanos() {
+    String query = String.format("SELECT date_nanos_value"
+        + " FROM %s", TEST_INDEX_DATATYPE_NONNUMERIC);
+    JSONObject result = executeQuery(query);
+    verifySchema(result,
+        schema("date_nanos_value", null, "timestamp"));
+    verifyDataRows(result,
+        rows("2019-03-24 01:34:46.123456789"));
+  }
+
   protected JSONObject executeQuery(String query) throws IOException {
     Request request = new Request("POST", QUERY_API_ENDPOINT);
     request.setJsonEntity(String.format(Locale.ROOT, "{\n" + "  \"query\": \"%s\"\n" + "}", query));

diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java
@@ -47,14 +47,14 @@ public void typeof_opensearch_types() {
     verifyDataRows(response,
         rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE"));
 
-    response = executeJdbcRequest(String.format("SELECT typeof(text_value),"
+    response = executeJdbcRequest(String.format("SELECT typeof(text_value), typeof(date_nanos_value),"
         + "typeof(date_value), typeof(boolean_value), typeof(object_value), typeof(keyword_value),"
         + "typeof(ip_value), typeof(binary_value), typeof(geo_point_value)"
         // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported
         //+ ", typeof(nested_value)"
         + " from %s;", TEST_INDEX_DATATYPE_NONNUMERIC));
     verifyDataRows(response,
-        rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
+        rows("TEXT", "TIMESTAMP", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
                 "IP", "BINARY", "GEO_POINT"));
   }
 }
diff --git a/integ-test/src/test/resources/datatypes.json b/integ-test/src/test/resources/datatypes.json
@@ -1,2 +1,2 @@
 {"index":{"_id":"1"}}
-{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" :  "Smith"}, {"first" : "Alice", "last" :  "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }}
+{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" :  "Smith"}, {"first" : "Alice", "last" :  "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }}
diff --git a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json
@@ -13,10 +13,13 @@
       "binary_value": {
         "type": "binary"
       },
-      "date_value":  {
+      "date_value": {
         "type" : "date",
         "format": "yyyy-MM-dd HH:mm:ss"
       },
+      "date_nanos_value": {
+        "type" : "date_nanos"
+      },
       "ip_value": {
         "type": "ip"
       },

diff --git a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json
@@ -298,7 +298,7 @@
         "format": "yyyy-MM-dd||epoch_millis"
       },
       "hour_minute_second_OR_t_time" : {
-        "type" : "date",
+        "type" : "date_nanos",
         "format": "hour_minute_second||t_time"
       },
       "custom_timestamp" : {

diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java
@@ -35,6 +35,7 @@ public enum MappingType {
     GeoPoint("geo_point", ExprCoreType.UNKNOWN),
     Binary("binary", ExprCoreType.UNKNOWN),
     Date("date", ExprCoreType.TIMESTAMP),
+    DateNanos("date_nanos", ExprCoreType.TIMESTAMP),
     Object("object", ExprCoreType.STRUCT),
     Nested("nested", ExprCoreType.ARRAY),
     Byte("byte", ExprCoreType.BYTE),
@@ -137,6 +138,7 @@ public static Map<String, OpenSearchDataType> parseMapping(Map<String, Object> i
    * @param mappingType A mapping type.
    * @return An instance or inheritor of `OpenSearchDataType`.
    */
+  @SuppressWarnings("unchecked")
   public static OpenSearchDataType of(MappingType mappingType, Map<String, Object> innerMap) {
     OpenSearchDataType res = instances.getOrDefault(mappingType.toString(),
         new OpenSearchDataType(mappingType)
@@ -162,6 +164,7 @@ public static OpenSearchDataType of(MappingType mappingType, Map<String, Object>
       case Binary: return OpenSearchBinaryType.of();
       case Ip: return OpenSearchIpType.of();
       case Date:
+      case DateNanos:
         // Default date formatter is used when "" is passed as the second parameter
         String format = (String) innerMap.getOrDefault("format", "");
         return OpenSearchDateType.of(format);

diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java
@@ -106,6 +106,7 @@ private static Stream<Arguments> getTestDataWithType() {
         Arguments.of(MappingType.Double, "double", DOUBLE),
         Arguments.of(MappingType.Boolean, "boolean", BOOLEAN),
         Arguments.of(MappingType.Date, "date", TIMESTAMP),
+        Arguments.of(MappingType.DateNanos, "date", TIMESTAMP),
         Arguments.of(MappingType.Object, "object", STRUCT),
         Arguments.of(MappingType.Nested, "nested", ARRAY),
         Arguments.of(MappingType.GeoPoint, "geo_point",