Add support for date_nanos and tests.

integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java

@@ -50,6 +50,7 @@ public void test_nonnumeric_data_types() throws IOException {
         schema("text_value", "string"),
         schema("binary_value", "binary"),
         schema("date_value", "timestamp"),
+        schema("date_nanos_value", "timestamp"),
         schema("ip_value", "ip"),
         schema("object_value", "struct"),
         schema("nested_value", "array"),

integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java

@@ -59,16 +59,16 @@ public void typeof_opensearch_types() throws IOException {
         rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE"));
 
     response = executeQuery(String.format("source=%s | eval "
-        + "`text` = typeof(text_value), `date` = typeof(date_value),"
+        + "`text` = typeof(text_value), `date` = typeof(date_value), `date_nanos` = typeof(date_nanos_value),"
         + "`boolean` = typeof(boolean_value), `object` = typeof(object_value),"
         + "`keyword` = typeof(keyword_value), `ip` = typeof(ip_value),"
         + "`binary` = typeof(binary_value), `geo_point` = typeof(geo_point_value)"
         // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported
         //+ ", `nested` = typeof(nested_value)"
-        + " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`",
+        + " | fields `text`, `date`, `date_nanos`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`",
             TEST_INDEX_DATATYPE_NONNUMERIC));
     verifyDataRows(response,
-        rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
+        rows("TEXT", "TIMESTAMP", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
                 "IP", "BINARY", "GEO_POINT"));
   }
 }

integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java

@@ -6,6 +6,7 @@
 
 package org.opensearch.sql.sql;
 
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS;
 import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT;
 import static org.opensearch.sql.util.MatcherUtils.rows;
@@ -30,6 +31,7 @@ public class DateTimeFormatsIT extends SQLIntegTestCase {
   public void init() throws Exception {
     super.init();
     loadIndex(Index.DATE_FORMATS);
+    loadIndex(Index.DATA_TYPE_NONNUMERIC);
   }
 
   @Test
@@ -120,6 +122,102 @@ public void testNumericFormats() {
         rows("1970-01-02 03:55:00", "1970-01-01 00:01:40.5"));
   }
 
+  @Test
+  @SneakyThrows
+  public void testDateNanosWithFormats() {
+    String query = String.format("SELECT hour_minute_second_OR_t_time"
+        + " FROM %s", TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result,
+        schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result,
+        rows("09:07:42"),
+        rows("07:07:42"));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosWithFunctions() {
+    // in memory funcs
+    String query = String.format(
+        "SELECT"
+            + " hour_minute_second_OR_t_time > TIME '08:07:00',"
+            + " hour_minute_second_OR_t_time < TIME '08:07:00',"
+            + " hour_minute_second_OR_t_time = t_time_no_millis,"
+            + " hour_minute_second_OR_t_time <> strict_t_time,"
+            + " hour_minute_second_OR_t_time >= t_time"
+            + " FROM %s",
+        TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result,
+        schema("hour_minute_second_OR_t_time > TIME '08:07:00'", null, "boolean"),
+        schema("hour_minute_second_OR_t_time < TIME '08:07:00'", null, "boolean"),
+        schema("hour_minute_second_OR_t_time = t_time_no_millis", null, "boolean"),
+        schema("hour_minute_second_OR_t_time <> strict_t_time", null, "boolean"),
+        schema("hour_minute_second_OR_t_time >= t_time", null, "boolean"));
+    verifyDataRows(result,
+        rows(true, false, true, false, true),
+        rows(false, true, false, true, false));
+    // push down
+    query = String.format(
+        "SELECT hour_minute_second_OR_t_time"
+            + " FROM %s WHERE hour_minute_second_OR_t_time > TIME '08:07:00'",
+        TEST_INDEX_DATE_FORMATS);
+    result = executeQuery(query);
+    verifySchema(result,
+        schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result,
+        rows("09:07:42"));
+    query = String.format(
+        "SELECT hour_minute_second_OR_t_time"
+            + " FROM %s WHERE hour_minute_second_OR_t_time < TIME '08:07:00'",
+        TEST_INDEX_DATE_FORMATS);
+    result = executeQuery(query);
+    verifySchema(result,
+        schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result,
+        rows("07:07:42"));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosOrderBy() {
+    String query = String.format("SELECT hour_minute_second_OR_t_time"
+        + " FROM %s ORDER BY hour_minute_second_OR_t_time ASC", TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result,
+        schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result,
+        rows("07:07:42"),
+        rows("09:07:42"));
+  }
+
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosGroupBy() {
+    String query = String.format("SELECT count(*)"
+        + " FROM %s GROUP BY hour_minute_second_OR_t_time", TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result,
+        schema("count(*)", null, "integer"));
+    verifyDataRows(result,
+        rows(1),
+        rows(1));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosWithNanos() {
+    String query = String.format("SELECT date_nanos_value"
+        + " FROM %s", TEST_INDEX_DATATYPE_NONNUMERIC);
+    JSONObject result = executeQuery(query);
+    verifySchema(result,
+        schema("date_nanos_value", null, "timestamp"));
+    verifyDataRows(result,
+        rows("2019-03-24 01:34:46.123456789"));
+  }
+
   protected JSONObject executeQuery(String query) throws IOException {
     Request request = new Request("POST", QUERY_API_ENDPOINT);
     request.setJsonEntity(String.format(Locale.ROOT, "{\n" + "  \"query\": \"%s\"\n" + "}", query));

integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java

@@ -47,14 +47,14 @@ public void typeof_opensearch_types() {
     verifyDataRows(response,
         rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE"));
 
-    response = executeJdbcRequest(String.format("SELECT typeof(text_value),"
+    response = executeJdbcRequest(String.format("SELECT typeof(text_value), typeof(date_nanos_value),"
         + "typeof(date_value), typeof(boolean_value), typeof(object_value), typeof(keyword_value),"
         + "typeof(ip_value), typeof(binary_value), typeof(geo_point_value)"
         // TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported
         //+ ", typeof(nested_value)"
         + " from %s;", TEST_INDEX_DATATYPE_NONNUMERIC));
     verifyDataRows(response,
-        rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
+        rows("TEXT", "TIMESTAMP", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD",
                 "IP", "BINARY", "GEO_POINT"));
   }
 }

integ-test/src/test/resources/datatypes.json

@@ -1,2 +1,2 @@
 {"index":{"_id":"1"}}
-{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" :  "Smith"}, {"first" : "Alice", "last" :  "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }}
+{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" :  "Smith"}, {"first" : "Alice", "last" :  "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }}

integ-test/src/test/resources/date_formats.json

@@ -1,4 +1,4 @@
 {"index": {}}
 {"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 AM", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42", "custom_timestamp": "1984-04-12 09:07:42 ---- AM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time":  "1961-04-12", "custom_time_parser_check":  "85476321", "incomplete_1" : 1984, "incomplete_2": null, "incomplete_custom_date": 1999, "incomplete_custom_time" : 10, "incorrect" : null, "epoch_sec" : 42, "epoch_milli" : 42, "custom_no_delimiter_date" : "19841020", "custom_no_delimiter_time" : "102030", "custom_no_delimiter_ts" : "19841020153548"}
 {"index": {}}
-{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time":  "09:07:00", "custom_time_parser_check":  "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incomplete_custom_date": 3021, "incomplete_custom_time" : 20, "incorrect" : null, "epoch_sec" : 100500, "epoch_milli" : 100500, "custom_no_delimiter_date" : "19610412", "custom_no_delimiter_time" : "090700", "custom_no_delimiter_ts" : "19610412090700"}
+{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T07:07:42.000Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time":  "09:07:00", "custom_time_parser_check":  "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incomplete_custom_date": 3021, "incomplete_custom_time" : 20, "incorrect" : null, "epoch_sec" : 100500, "epoch_milli" : 100500, "custom_no_delimiter_date" : "19610412", "custom_no_delimiter_time" : "090700", "custom_no_delimiter_ts" : "19610412090700"}

integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json

@@ -13,10 +13,13 @@
       "binary_value": {
         "type": "binary"
       },
-      "date_value":  {
+      "date_value": {
         "type" : "date",
         "format": "yyyy-MM-dd HH:mm:ss"
       },
+      "date_nanos_value": {
+        "type" : "date_nanos"
+      },
       "ip_value": {
         "type": "ip"
       },

integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json

@@ -298,7 +298,7 @@
         "format": "yyyy-MM-dd||epoch_millis"
       },
       "hour_minute_second_OR_t_time" : {
-        "type" : "date",
+        "type" : "date_nanos",
         "format": "hour_minute_second||t_time"
       },
       "custom_timestamp" : {