From 679c3e90b12ed34625a4aeac37a21e166ce6beb5 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Mon, 16 Dec 2024 10:18:12 +0800 Subject: [PATCH 01/23] add flex proto --- .../executor/ir/proto/flex_type.proto | 119 ++++++++++++++++++ .../executor/ir/proto/type.proto | 2 + 2 files changed, 121 insertions(+) create mode 100644 interactive_engine/executor/ir/proto/flex_type.proto diff --git a/interactive_engine/executor/ir/proto/flex_type.proto b/interactive_engine/executor/ir/proto/flex_type.proto new file mode 100644 index 000000000000..9fd2dacdc507 --- /dev/null +++ b/interactive_engine/executor/ir/proto/flex_type.proto @@ -0,0 +1,119 @@ +syntax = "proto3"; + +package flex; +option java_package = "com.alibaba.graphscope.proto.type"; + +enum PrimitiveType { + DT_ANY = 0; + DT_SIGNED_INT32 = 1; + DT_UNSIGNED_INT32 = 2; + DT_SIGNED_INT64 = 3; + DT_UNSIGNED_INT64 = 4; + DT_BOOL = 5; + DT_FLOAT = 6; + DT_DOUBLE = 7; +} + +message Decimal { // precision=4 scale=2 : 23.12 + uint32 precision = 1; + uint32 scale = 2; +} + +message String { + // string with unlimited length + message LongText { + } + + // string with fixed length + message Char { + uint32 fixed_length = 1; + } + + // string with variable length, bounded by max_length + message VarChar { + uint32 max_length = 1; + } + + oneof item { + LongText long_text = 1; + Char char = 2; + VarChar var_char = 3; + } +} + +// temporal types + +message Temporal { + enum DateFormat { + DF_YYYY_MM_DD = 0; // ISO fomat: 2019-01-01 + } + + enum TimeFormat { + TF_HH_MM_SS_SSS = 0; // ISO format: 00:00:00.000 + } + + enum DateTimeFormat { + DTF_YYYY_MM_DD_HH_MM_SS_SSS = 0; // ISO format: 2019-01-01 00:00:00.000 + } + + enum TimeZoneFormat { + TZF_UTC = 0; // Z + TZF_OFFSET = 1; // +08:00 or -08:00 + } + + message Date { + DateFormat date_format = 1; + } + + message Time { + TimeFormat time_format = 1; + TimeZoneFormat time_zone_format = 2; + } + + message DateTime { + DateTimeFormat date_time_format = 1; + TimeZoneFormat time_zone_format = 2; + } + + // int32 days since 1970-01-01 + message Date32 { + } + + // int32 milliseconds past midnight + message Time32 { + } + + // int64 milliseconds since 1970-01-01 00:00:00.000000 + message Timestamp { + } + + oneof item { + Date date = 1; + Time time = 2; + DateTime date_time = 3; + Date32 date32 = 4; + Time32 time32 = 5; + Timestamp timestamp = 6; + } +} + +message Array { + StorageDataType component_type = 1; + uint32 max_length = 2; +} + +message Map { + StorageDataType key_type = 1; + StorageDataType value_type = 2; +} + +message StorageDataType { + oneof item { + PrimitiveType primitive_type = 1; + Decimal decimal = 2; + String string = 3; + Temporal temporal = 4; + Array array = 5; + Map map = 6; + } +} diff --git a/interactive_engine/executor/ir/proto/type.proto b/interactive_engine/executor/ir/proto/type.proto index 3494e7eea13d..4f2778a0f415 100644 --- a/interactive_engine/executor/ir/proto/type.proto +++ b/interactive_engine/executor/ir/proto/type.proto @@ -20,6 +20,7 @@ option java_package = "com.alibaba.graphscope.gaia.proto"; option java_outer_classname = "DataType"; import "common.proto"; +import "flex_type.proto"; import "google/protobuf/wrappers.proto"; @@ -61,5 +62,6 @@ message IrDataType { oneof type { common.DataType data_type = 1; GraphDataType graph_type = 2; + flex.StorageDataType flex_type = 3; } } From 36f5b29ed543c6fb0772181c021bfa491a3fc837 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Mon, 16 Dec 2024 12:47:04 +0800 Subject: [PATCH 02/23] fix data type proto --- .../{flex_type.proto => basic_type.proto} | 10 +++++----- .../executor/ir/proto/common.proto | 18 ------------------ .../executor/ir/proto/schema.proto | 1 + .../executor/ir/proto/type.proto | 6 ++---- 4 files changed, 8 insertions(+), 27 deletions(-) rename interactive_engine/executor/ir/proto/{flex_type.proto => basic_type.proto} (93%) diff --git a/interactive_engine/executor/ir/proto/flex_type.proto b/interactive_engine/executor/ir/proto/basic_type.proto similarity index 93% rename from interactive_engine/executor/ir/proto/flex_type.proto rename to interactive_engine/executor/ir/proto/basic_type.proto index 9fd2dacdc507..4ff14bf771f7 100644 --- a/interactive_engine/executor/ir/proto/flex_type.proto +++ b/interactive_engine/executor/ir/proto/basic_type.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package flex; +package common; option java_package = "com.alibaba.graphscope.proto.type"; enum PrimitiveType { @@ -98,16 +98,16 @@ message Temporal { } message Array { - StorageDataType component_type = 1; + DataType component_type = 1; uint32 max_length = 2; } message Map { - StorageDataType key_type = 1; - StorageDataType value_type = 2; + DataType key_type = 1; + DataType value_type = 2; } -message StorageDataType { +message DataType { oneof item { PrimitiveType primitive_type = 1; Decimal decimal = 2; diff --git a/interactive_engine/executor/ir/proto/common.proto b/interactive_engine/executor/ir/proto/common.proto index 034c6581f74a..14a07f03da4f 100644 --- a/interactive_engine/executor/ir/proto/common.proto +++ b/interactive_engine/executor/ir/proto/common.proto @@ -89,21 +89,3 @@ message Value { Timestamp timestamp = 16; } } - -enum DataType { - BOOLEAN = 0; - INT32 = 1; - INT64 = 2 ; - DOUBLE = 3; - STRING = 4; - BYTES = 5; - INT32_ARRAY = 6; - INT64_ARRAY = 7; - DOUBLE_ARRAY = 8; - STRING_ARRAY = 9; - PAIR_ARRAY = 10; - NONE = 11; - DATE32 = 12; - TIME32 = 13; - TIMESTAMP = 14; -} diff --git a/interactive_engine/executor/ir/proto/schema.proto b/interactive_engine/executor/ir/proto/schema.proto index 18bdafe759ee..6cd7b58ef97a 100644 --- a/interactive_engine/executor/ir/proto/schema.proto +++ b/interactive_engine/executor/ir/proto/schema.proto @@ -21,6 +21,7 @@ option java_package = "com.alibaba.graphscope.gaia.proto"; option java_outer_classname = "OuterSchema"; import "common.proto"; +import "basic_type.proto"; message LabelMeta { int32 id = 1; diff --git a/interactive_engine/executor/ir/proto/type.proto b/interactive_engine/executor/ir/proto/type.proto index 4f2778a0f415..0e2282334120 100644 --- a/interactive_engine/executor/ir/proto/type.proto +++ b/interactive_engine/executor/ir/proto/type.proto @@ -20,10 +20,9 @@ option java_package = "com.alibaba.graphscope.gaia.proto"; option java_outer_classname = "DataType"; import "common.proto"; -import "flex_type.proto"; +import "basic_type.proto"; import "google/protobuf/wrappers.proto"; - // GraphDataType, including structured data types of vertices or edges. message GraphDataType { // The Option of the graph element, i.e., vertex or edge @@ -59,9 +58,8 @@ message GraphDataType { // IrDataType, which could be either basic DataType or GraphDataType message IrDataType { - oneof type { + oneof item { common.DataType data_type = 1; GraphDataType graph_type = 2; - flex.StorageDataType flex_type = 3; } } From c22062629e732075d9c73d218b60d25ef2aa3c64 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Mon, 16 Dec 2024 12:50:35 +0800 Subject: [PATCH 03/23] minor fix --- interactive_engine/executor/ir/proto/basic_type.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/interactive_engine/executor/ir/proto/basic_type.proto b/interactive_engine/executor/ir/proto/basic_type.proto index 4ff14bf771f7..b7436664c64a 100644 --- a/interactive_engine/executor/ir/proto/basic_type.proto +++ b/interactive_engine/executor/ir/proto/basic_type.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package common; option java_package = "com.alibaba.graphscope.proto.type"; +option java_outer_classname = "Common"; enum PrimitiveType { DT_ANY = 0; From 1c95c4fd0671535d4c68f833e211e3283a9b24ce Mon Sep 17 00:00:00 2001 From: shirly121 Date: Mon, 16 Dec 2024 12:56:40 +0800 Subject: [PATCH 04/23] minor fix --- .../executor/ir/proto/basic_type.proto | 16 ++++++++-------- interactive_engine/executor/ir/proto/type.proto | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/interactive_engine/executor/ir/proto/basic_type.proto b/interactive_engine/executor/ir/proto/basic_type.proto index b7436664c64a..8e1d72d45c24 100644 --- a/interactive_engine/executor/ir/proto/basic_type.proto +++ b/interactive_engine/executor/ir/proto/basic_type.proto @@ -15,7 +15,7 @@ enum PrimitiveType { DT_DOUBLE = 7; } -message Decimal { // precision=4 scale=2 : 23.12 +message Decimal { // precision=4 scale=2 : 23.12 uint32 precision = 1; uint32 scale = 2; } @@ -29,7 +29,7 @@ message String { message Char { uint32 fixed_length = 1; } - + // string with variable length, bounded by max_length message VarChar { uint32 max_length = 1; @@ -48,29 +48,29 @@ message Temporal { enum DateFormat { DF_YYYY_MM_DD = 0; // ISO fomat: 2019-01-01 } - + enum TimeFormat { TF_HH_MM_SS_SSS = 0; // ISO format: 00:00:00.000 } - + enum DateTimeFormat { DTF_YYYY_MM_DD_HH_MM_SS_SSS = 0; // ISO format: 2019-01-01 00:00:00.000 } - + enum TimeZoneFormat { TZF_UTC = 0; // Z TZF_OFFSET = 1; // +08:00 or -08:00 } - + message Date { DateFormat date_format = 1; } - + message Time { TimeFormat time_format = 1; TimeZoneFormat time_zone_format = 2; } - + message DateTime { DateTimeFormat date_time_format = 1; TimeZoneFormat time_zone_format = 2; diff --git a/interactive_engine/executor/ir/proto/type.proto b/interactive_engine/executor/ir/proto/type.proto index 0e2282334120..6f258ee9a8fa 100644 --- a/interactive_engine/executor/ir/proto/type.proto +++ b/interactive_engine/executor/ir/proto/type.proto @@ -58,7 +58,7 @@ message GraphDataType { // IrDataType, which could be either basic DataType or GraphDataType message IrDataType { - oneof item { + oneof type { common.DataType data_type = 1; GraphDataType graph_type = 2; } From d3b04abaf7cb5949dd77a9685d5e3ba2bfffe0f9 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Mon, 16 Dec 2024 14:03:46 +0800 Subject: [PATCH 05/23] build for the basic_type proto --- interactive_engine/executor/ir/common/build.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/interactive_engine/executor/ir/common/build.rs b/interactive_engine/executor/ir/common/build.rs index 96cb3a96dea9..a0b395e72db8 100644 --- a/interactive_engine/executor/ir/common/build.rs +++ b/interactive_engine/executor/ir/common/build.rs @@ -32,6 +32,7 @@ fn codegen_inplace() -> Result<(), Box> { println!("cargo:rerun-if-changed=../proto/physical.proto"); println!("cargo:rerun-if-changed=../proto/type.proto"); println!("cargo:rerun-if-changed=../proto/stored_procedure.proto"); + println!("cargo:rerun-if-changed=../proto/basic_type.proto"); let out_dir = PathBuf::from(GEN_DIR); if out_dir.exists() { let _ = std::fs::remove_dir_all(GEN_DIR); @@ -50,6 +51,7 @@ fn codegen_inplace() -> Result<(), Box> { "../proto/physical.proto", "../proto/type.proto", "../proto/stored_procedure.proto", + "../proto/basic_type.proto", ], &["../proto"], )?; @@ -67,6 +69,7 @@ fn codegen_inplace() -> Result<(), Box> { println!("cargo:rerun-if-changed=../proto/physical.proto"); println!("cargo:rerun-if-changed=../proto/type.proto"); println!("cargo:rerun-if-changed=../proto/stored_procedure.proto"); + println!("cargo:rerun-if-changed=../proto/basic_type.proto"); prost_build::Config::new() .type_attribute(".", "#[derive(Serialize,Deserialize)]") .compile_protos( @@ -79,6 +82,7 @@ fn codegen_inplace() -> Result<(), Box> { "../proto/physical.proto", "../proto/type.proto", "../proto/stored_procedure.proto", + "../proto/basic_type.proto", ], &["../proto"], )?; From 72c5eb51b945270fee54ae40f1bf5958b1fc0152 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Mon, 16 Dec 2024 14:03:57 +0800 Subject: [PATCH 06/23] remove data type from schema.proto in ir-core --- interactive_engine/executor/ir/proto/schema.proto | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/interactive_engine/executor/ir/proto/schema.proto b/interactive_engine/executor/ir/proto/schema.proto index 6cd7b58ef97a..22d28ea664c8 100644 --- a/interactive_engine/executor/ir/proto/schema.proto +++ b/interactive_engine/executor/ir/proto/schema.proto @@ -30,9 +30,8 @@ message LabelMeta { message ColumnMeta { LabelMeta key = 1; - common.DataType data_type = 2; // Whether this column is a part of a primary key - bool is_primary_key = 3; + bool is_primary_key = 2; } message EntityMeta { From a5fc017be4758eeafaab83af2ca0a03d5bbff204 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 16 Dec 2024 19:56:29 +0800 Subject: [PATCH 07/23] adapt interactive to ir_align_type Committed-by: xiaolei.zl from Dev container --- flex/codegen/src/codegen_utils.h | 19 +- flex/codegen/src/graph_types.h | 117 ++++--- .../codegen/src/hqps/hqps_case_when_builder.h | 5 +- .../src/hqps/hqps_edge_expand_builder.h | 4 +- flex/codegen/src/hqps/hqps_get_v_builder.h | 4 +- flex/codegen/src/hqps/hqps_scan_builder.h | 4 +- flex/codegen/src/hqps/hqps_select_builder.h | 4 +- .../src/pegasus/pegasus_order_by_builder.h | 29 +- .../src/pegasus/pegasus_project_builder.h | 24 +- .../graph_db/runtime/adhoc/expr_impl.cc | 40 ++- .../graph_db/runtime/adhoc/operators/scan.cc | 31 +- flex/engines/graph_db/runtime/adhoc/utils.cc | 331 ++++++++++-------- .../engines/graph_db/runtime/common/rt_any.cc | 45 ++- flex/utils/CMakeLists.txt | 1 + 14 files changed, 404 insertions(+), 254 deletions(-) diff --git a/flex/codegen/src/codegen_utils.h b/flex/codegen/src/codegen_utils.h index 563cd6544521..a8ab552edf51 100644 --- a/flex/codegen/src/codegen_utils.h +++ b/flex/codegen/src/codegen_utils.h @@ -132,10 +132,23 @@ std::string generate_output_list(std::string input_name, int32_t input_size, // check type consistent bool data_type_consistent(const common::DataType& left, const common::DataType& right) { - if (left == common::DataType::NONE || right == common::DataType::NONE) { - return true; + if (left.item_case() == common::DataType::ITEM_NOT_SET) { + return false; + } + if (left.item_case() != right.item_case()) { + return false; + } + if (left.item_case() == common::DataType::kPrimitiveType) { + return left.primitive_type() == right.primitive_type(); + } else if (left.item_case() == common::DataType::kArray || + left.item_case() == common::DataType::kMap) { + LOG(FATAL) << "Not support list or map type"; + } else if (left.item_case() == common::DataType::kString) { + return true; // string type is always consistent + } else { + LOG(FATAL) << "Unexpected data type"; + return false; } - return left == right; } std::tuple decode_param_from_decoder( diff --git a/flex/codegen/src/graph_types.h b/flex/codegen/src/graph_types.h index fa0316b399a4..c9e45c2e2df5 100644 --- a/flex/codegen/src/graph_types.h +++ b/flex/codegen/src/graph_types.h @@ -19,6 +19,7 @@ limitations under the License. #include #include "flex/codegen/src/string_utils.h" +#include "flex/proto_generated_gie/basic_type.pb.h" #include "flex/proto_generated_gie/common.pb.h" #include "glog/logging.h" #include "google/protobuf/any.h" @@ -63,62 +64,93 @@ inline bool operator==(const ParamConst& lhs, const ParamConst& rhs) { } // namespace codegen -static codegen::DataType common_data_type_pb_2_data_type( - const common::DataType& data_type) { - switch (data_type) { - case common::DataType::INT32: +static codegen::DataType primitive_type_to_data_type( + const common::PrimitiveType& type) { + switch (type) { + case common::PrimitiveType::DT_SIGNED_INT32: return codegen::DataType::kInt32; - case common::DataType::INT64: + case common::PrimitiveType::DT_SIGNED_INT64: return codegen::DataType::kInt64; - case common::DataType::DOUBLE: + case common::PrimitiveType::DT_FLOAT: + return codegen::DataType::kFloat; + case common::PrimitiveType::DT_DOUBLE: return codegen::DataType::kDouble; - case common::DataType::STRING: - return codegen::DataType::kString; - case common::DataType::INT64_ARRAY: - return codegen::DataType::kInt64Array; - case common::DataType::INT32_ARRAY: - return codegen::DataType::kInt32Array; - case common::DataType::BOOLEAN: + case common::PrimitiveType::DT_BOOL: return codegen::DataType::kBoolean; - case common::DataType::DATE32: - return codegen::DataType::kDate; - case common::DataType::TIME32: - return codegen::DataType::kTime; - case common::DataType::TIMESTAMP: - return codegen::DataType::kTimeStamp; + default: + // LOG(FATAL) << "unknown primitive type"; + throw std::runtime_error( + "unknown primitive type when converting primitive type to data type:" + + std::to_string(static_cast(type))); + } +} + +static codegen::DataType common_data_type_pb_2_data_type( + const common::DataType& data_type) { + switch (data_type.item_case()) { + case common::DataType::ItemCase::kPrimitiveType: + return primitive_type_to_data_type(data_type.primitive_type()); + case common::DataType::ItemCase::kDecimal: + LOG(FATAL) << "Not support decimal type"; + case common::DataType::ItemCase::kString: + return codegen::DataType::kString; + case common::DataType::ItemCase::kTemporal: + LOG(FATAL) << "Not support temporal type"; + case common::DataType::ItemCase::kArray: + case common::DataType::ItemCase::kMap: + LOG(FATAL) << "Not support array or map type"; default: // LOG(FATAL) << "unknown data type"; throw std::runtime_error( "unknown data type when converting common_data_type to inner data " "type:" + - std::to_string(static_cast(data_type))); + data_type.DebugString()); } } -static std::string single_common_data_type_pb_2_str( - const common::DataType& data_type) { - switch (data_type) { - case common::DataType::BOOLEAN: - return "bool"; - case common::DataType::INT32: +static std::string primitive_type_to_str(const common::PrimitiveType& type) { + switch (type) { + case common::PrimitiveType::DT_SIGNED_INT32: return "int32_t"; - case common::DataType::INT64: + case common::PrimitiveType::DT_UNSIGNED_INT32: + return "uint32_t"; + case common::PrimitiveType::DT_SIGNED_INT64: return "int64_t"; - case common::DataType::DOUBLE: + case common::PrimitiveType::DT_UNSIGNED_INT64: + return "uint64_t"; + case common::PrimitiveType::DT_FLOAT: + return "float"; + case common::PrimitiveType::DT_DOUBLE: return "double"; - case common::DataType::STRING: + case common::PrimitiveType::DT_BOOL: + return "bool"; + default: + // LOG(FATAL) << "unknown primitive type"; + throw std::runtime_error( + "unknown primitive type when converting primitive type to string:" + + std::to_string(static_cast(type))); + } +} + +static std::string single_common_data_type_pb_2_str( + const common::DataType& data_type) { + switch (data_type.item_case()) { + case common::DataType::ItemCase::kPrimitiveType: + return primitive_type_to_str(data_type.primitive_type()); + case common::DataType::ItemCase::kDecimal: + LOG(FATAL) << "Not support decimal type"; + case common::DataType::ItemCase::kString: return "std::string_view"; - case common::DataType::INT64_ARRAY: - return "std::vector"; - case common::DataType::INT32_ARRAY: - return "std::vector"; - case common::DataType::DATE32: - return "Date"; + case common::DataType::ItemCase::kTemporal: + LOG(FATAL) << "Not support temporal type"; + case common::DataType::ItemCase::kArray: + case common::DataType::ItemCase::kMap: + LOG(FATAL) << "Not support array or map type"; // TODO: support time32 and timestamp default: throw std::runtime_error( "unknown data type when convert common data type to string:" + - std::to_string(static_cast(data_type))); + data_type.DebugString()); } } @@ -266,21 +298,22 @@ static std::string data_type_2_rust_string(const codegen::DataType& data_type) { } static common::DataType common_value_2_data_type(const common::Value& value) { + common::DataType ret; switch (value.item_case()) { case common::Value::kI32: - return common::DataType::INT32; + ret.set_primitive_type(common::PrimitiveType::DT_SIGNED_INT32); case common::Value::kI64: - return common::DataType::INT64; + ret.set_primitive_type(common::PrimitiveType::DT_SIGNED_INT64); case common::Value::kBoolean: - return common::DataType::BOOLEAN; + ret.set_primitive_type(common::PrimitiveType::DT_BOOL); case common::Value::kF64: - return common::DataType::DOUBLE; + ret.set_primitive_type(common::PrimitiveType::DT_DOUBLE); case common::Value::kStr: - return common::DataType::STRING; + ret.mutable_string()->mutable_long_text(); default: LOG(FATAL) << "unknown value" << value.DebugString(); } - return common::DataType::NONE; + return ret; } static void parse_param_const_from_pb( diff --git a/flex/codegen/src/hqps/hqps_case_when_builder.h b/flex/codegen/src/hqps/hqps_case_when_builder.h index f3ea57a16b21..9c867fee5cb9 100644 --- a/flex/codegen/src/hqps/hqps_case_when_builder.h +++ b/flex/codegen/src/hqps/hqps_case_when_builder.h @@ -118,9 +118,8 @@ class CaseWhenBuilder : public ExprBuilder { std::string str = formater.str(); - return std::make_tuple( - class_name_, construct_params_, tag_selectors_, str, - std::vector{common::DataType::DataType_INT_MIN_SENTINEL_DO_NOT_USE_}); + return std::make_tuple(class_name_, construct_params_, tag_selectors_, str, + std::vector{common::DataType()}); } protected: diff --git a/flex/codegen/src/hqps/hqps_edge_expand_builder.h b/flex/codegen/src/hqps/hqps_edge_expand_builder.h index 9a2b678d7730..0deb25f41f4a 100644 --- a/flex/codegen/src/hqps/hqps_edge_expand_builder.h +++ b/flex/codegen/src/hqps/hqps_edge_expand_builder.h @@ -224,7 +224,9 @@ static void BuildExprFromPredicate(BuildingContext& ctx, std::string& func_construct_params_str, std::string& property_selectors_str) { auto expr_builder = ExprBuilder(ctx); - expr_builder.set_return_type(common::DataType::BOOLEAN); + common::DataType type; + type.set_primitive_type(common::PrimitiveType::DT_BOOL); + expr_builder.set_return_type(type); expr_builder.AddAllExprOpr(expr.operators()); std::string expr_code; std::vector func_call_param_const; diff --git a/flex/codegen/src/hqps/hqps_get_v_builder.h b/flex/codegen/src/hqps/hqps_get_v_builder.h index e9c7b95231c6..202b2ffcdff2 100644 --- a/flex/codegen/src/hqps/hqps_get_v_builder.h +++ b/flex/codegen/src/hqps/hqps_get_v_builder.h @@ -135,7 +135,9 @@ class GetVOpBuilder { auto& expr_oprs = expr.operators(); expr_builder.AddAllExprOpr(expr_oprs); - expr_builder.set_return_type(common::DataType::BOOLEAN); + common::DataType data_type; + data_type.set_primitive_type(common::PrimitiveType::DT_BOOL); + expr_builder.set_return_type(data_type); std::vector unused_expr_ret_type; if (!expr_builder.empty()) { std::tie(expr_name_, expr_call_param_, tag_properties_, expr_code_, diff --git a/flex/codegen/src/hqps/hqps_scan_builder.h b/flex/codegen/src/hqps/hqps_scan_builder.h index cf210adf30f8..93df39b9ae57 100644 --- a/flex/codegen/src/hqps/hqps_scan_builder.h +++ b/flex/codegen/src/hqps/hqps_scan_builder.h @@ -149,7 +149,9 @@ class ScanOpBuilder { // TODO: make expr_builder a member of ScanOpBuilder // auto expr_builder = ExprBuilder(ctx_); - expr_builder_.set_return_type(common::DataType::BOOLEAN); + common::DataType type; + type.set_primitive_type(common::PrimitiveType::DT_BOOL); + expr_builder_.set_return_type(common::DataType(type)); // Add extra (, ) to wrap the code, since we may append index_predicate // afterwards. common::ExprOpr left_brace, right_brace; diff --git a/flex/codegen/src/hqps/hqps_select_builder.h b/flex/codegen/src/hqps/hqps_select_builder.h index 4653f24e24a5..3bdd68867437 100644 --- a/flex/codegen/src/hqps/hqps_select_builder.h +++ b/flex/codegen/src/hqps/hqps_select_builder.h @@ -41,7 +41,9 @@ class SelectOpBuilder { SelectOpBuilder& expr(const common::Expression expr) { ExprBuilder expr_builder(ctx_); - expr_builder.set_return_type(common::DataType::BOOLEAN); + common::DataType data_type; + data_type.set_primitive_type(common::PrimitiveType::DT_BOOL); + expr_builder.set_return_type(data_type); expr_builder.AddAllExprOpr(expr.operators()); std::string func_code; diff --git a/flex/codegen/src/pegasus/pegasus_order_by_builder.h b/flex/codegen/src/pegasus/pegasus_order_by_builder.h index d948c07fcd5c..e799da4c2b6e 100644 --- a/flex/codegen/src/pegasus/pegasus_order_by_builder.h +++ b/flex/codegen/src/pegasus/pegasus_order_by_builder.h @@ -72,20 +72,29 @@ class OrderByOpBuilder { ss << ".then("; } std::string cmp_type; - switch (data_type) { - case common::DataType::BOOLEAN: - case common::DataType::INT32: - case common::DataType::INT64: - case common::DataType::STRING: { - cmp_type = "cmp"; - break; + switch (data_type.item_case()) { + case common::DataType::kPrimitiveType: { + switch (data_type.primitive_type()) { + case common::PrimitiveType::DT_BOOL: + case common::PrimitiveType::DT_SIGNED_INT32: + case common::PrimitiveType::DT_SIGNED_INT64: + cmp_type = "cmp"; + break; + case common::PrimitiveType::DT_DOUBLE: { + cmp_type = "partial_cmp"; + break; + } + default: + LOG(FATAL) << "Unsupported type " + << static_cast(data_type.primitive_type()); + } } - case common::DataType::DOUBLE: { - cmp_type = "partial_cmp"; + case common::DataType::kString: { + cmp_type = "cmp"; break; } default: - LOG(FATAL) << "Unsupported type " << data_type; + LOG(FATAL) << "Unsupported type " << data_type.DebugString(); } std::string reverse_str; if (ordering_pair_[i].order() == algebra::OrderBy_OrderingPair_Order:: diff --git a/flex/codegen/src/pegasus/pegasus_project_builder.h b/flex/codegen/src/pegasus/pegasus_project_builder.h index 7ff5d4883821..f0595aa15797 100644 --- a/flex/codegen/src/pegasus/pegasus_project_builder.h +++ b/flex/codegen/src/pegasus/pegasus_project_builder.h @@ -176,14 +176,24 @@ class ProjectOpBuilder { ctx_.SetOutput(i, data_types); } else if (column_meta.type().type_case() == common::IrDataType::kDataType) { - switch (column_meta.type().data_type()) { - case common::DataType::INT64: { - std::vector data_types; - data_types.push_back(codegen::DataType::kInt64); - ctx_.SetOutput(i, data_types); - break; + switch (column_meta.type().data_type().item_case()) { + case common::DataType::kPrimitiveType: { + auto data_type = column_meta.type().data_type().primitive_type(); + switch (data_type) { + case common::PrimitiveType::DT_SIGNED_INT64: { + std::vector data_types; + data_types.push_back(codegen::DataType::kInt64); + ctx_.SetOutput(i, data_types); + break; + } + default: { + std::vector data_types; + data_types.push_back(codegen::DataType::kString); + ctx_.SetOutput(i, data_types); + } + } } - case common::DataType::STRING: { + case common::DataType::kString: { std::vector data_types; data_types.push_back(codegen::DataType::kString); ctx_.SetOutput(i, data_types); diff --git a/flex/engines/graph_db/runtime/adhoc/expr_impl.cc b/flex/engines/graph_db/runtime/adhoc/expr_impl.cc index 7df4cdbeed15..b965f6045fcf 100644 --- a/flex/engines/graph_db/runtime/adhoc/expr_impl.cc +++ b/flex/engines/graph_db/runtime/adhoc/expr_impl.cc @@ -16,6 +16,7 @@ #include "flex/engines/graph_db/runtime/adhoc/expr_impl.h" #include #include +#include "flex/proto_generated_gie/basic_type.pb.h" namespace gs { @@ -441,21 +442,36 @@ static RTAny parse_param(const common::DynamicParam& param, common::IrDataType::TypeCase::kDataType) { common::DataType dt = param.data_type().data_type(); const std::string& name = param.name(); - if (dt == common::DataType::DATE32) { - int64_t val = std::stoll(input.at(name)); - return RTAny::from_int64(val); - } else if (dt == common::DataType::STRING) { + if (dt.item_case() == common::DataType::ItemCase::kPrimitiveType) { + switch (dt.primitive_type()) { + case common::PrimitiveType::DT_SIGNED_INT32: { + int val = std::stoi(input.at(name)); + return RTAny::from_int32(val); + } + case common::PrimitiveType::DT_SIGNED_INT64: { + int64_t val = std::stoll(input.at(name)); + return RTAny::from_int64(val); + } + case common::PrimitiveType::DT_DOUBLE: + return RTAny::from_double(std::stod(input.at(name))); + case common::PrimitiveType::DT_BOOL: + return RTAny::from_bool(input.at(name) == "true"); + default: + LOG(FATAL) << "not support type: " << dt.DebugString(); + } + } else if (dt.item_case() == common::DataType::ItemCase::kTemporal) { + if (dt.temporal().item_case() == common::Temporal::kDate32) { + int64_t val = std::stoll(input.at(name)); + return RTAny::from_int64(val); + } else { + LOG(FATAL) << "not support type: " << dt.temporal().DebugString(); + } + } else if (dt.item_case() == common::DataType::ItemCase::kString) { const std::string& val = input.at(name); return RTAny::from_string(val); - } else if (dt == common::DataType::INT32) { - int val = std::stoi(input.at(name)); - return RTAny::from_int32(val); - } else if (dt == common::DataType::INT64) { - int64_t val = std::stoll(input.at(name)); - return RTAny::from_int64(val); + } else { + LOG(FATAL) << "not support type: " << dt.DebugString(); } - - LOG(FATAL) << "not support type: " << common::DataType_Name(dt); } LOG(FATAL) << "graph data type not expected...."; return RTAny(); diff --git a/flex/engines/graph_db/runtime/adhoc/operators/scan.cc b/flex/engines/graph_db/runtime/adhoc/operators/scan.cc index d9cff3bfa4de..2dd417155427 100644 --- a/flex/engines/graph_db/runtime/adhoc/operators/scan.cc +++ b/flex/engines/graph_db/runtime/adhoc/operators/scan.cc @@ -16,6 +16,7 @@ #include "flex/engines/graph_db/runtime/common/operators/scan.h" #include "flex/engines/graph_db/runtime/adhoc/expr_impl.h" #include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/proto_generated_gie/basic_type.pb.h" namespace gs { namespace runtime { @@ -202,22 +203,28 @@ bool parse_idx_predicate(const algebra::IndexPredicate& predicate, const common::DynamicParam& p = triplet.param(); if (p.data_type().type_case() == common::IrDataType::TypeCase::kDataType) { auto dt = p.data_type().data_type(); - if (dt == common::DataType::INT64) { - std::string name = p.name(); - std::string value = params.at(name); - int64_t v = std::stoll(value); - oids.emplace_back(v); - } else if (dt == common::DataType::STRING) { + if (dt.item_case() == common::DataType::ItemCase::kPrimitiveType) { + if (dt.primitive_type() == common::PrimitiveType::DT_SIGNED_INT64) { + std::string name = p.name(); + std::string value = params.at(name); + int64_t v = std::stoll(value); + oids.emplace_back(v); + } else if (dt.primitive_type() == + common::PrimitiveType::DT_SIGNED_INT32) { + std::string name = p.name(); + std::string value = params.at(name); + int32_t v = std::stoi(value); + oids.emplace_back(v); + } else { + LOG(FATAL) << "unsupported primary key type" << dt.DebugString(); + return false; + } + } else if (dt.item_case() == common::DataType::ItemCase::kString) { std::string name = p.name(); std::string value = params.at(name); oids.emplace_back(Any::From(value)); - } else if (dt == common::DataType::INT32) { - std::string name = p.name(); - std::string value = params.at(name); - int32_t v = std::stoi(value); - oids.emplace_back(v); } else { - LOG(FATAL) << "unsupported primary key type" << dt; + LOG(FATAL) << "unsupported primary key type" << dt.item_case(); return false; } } diff --git a/flex/engines/graph_db/runtime/adhoc/utils.cc b/flex/engines/graph_db/runtime/adhoc/utils.cc index bd5135b0d2ca..473b5a0c9182 100644 --- a/flex/engines/graph_db/runtime/adhoc/utils.cc +++ b/flex/engines/graph_db/runtime/adhoc/utils.cc @@ -15,6 +15,7 @@ #include "flex/engines/graph_db/runtime/adhoc/utils.h" #include "flex/engines/graph_db/runtime/common/columns/value_columns.h" #include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" +#include "flex/proto_generated_gie/basic_type.pb.h" namespace gs { @@ -144,68 +145,82 @@ std::shared_ptr create_column_builder(RTAnyType type) { return nullptr; } -std::shared_ptr build_optional_column( - const common::IrDataType& data_type, const Expr& expr, size_t row_num) { - switch (data_type.type_case()) { - case common::IrDataType::kDataType: { - switch (data_type.data_type()) { - case common::DataType::INT64: { - OptionalValueColumnBuilder builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - auto v = expr.eval_path(i, 0); - if (v.is_null()) { - builder.push_back_null(); - } else { - builder.push_back_opt(v.as_int64(), true); - } +std::shared_ptr build_optional_primitive_column( + const common::PrimitiveType& data_type, const Expr& expr, size_t row_num) { + switch (data_type) { + case common::PrimitiveType::DT_SIGNED_INT64: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_int64(), true); } + } - return builder.finish(); - } break; - case common::DataType::INT32: { - OptionalValueColumnBuilder builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - auto v = expr.eval_path(i, 0); - if (v.is_null()) { - builder.push_back_null(); - } else { - builder.push_back_opt(v.as_int32(), true); - } + return builder.finish(); + } + case common::PrimitiveType::DT_SIGNED_INT32: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_int32(), true); } + } - return builder.finish(); - } break; - case common::DataType::DOUBLE: { - OptionalValueColumnBuilder builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - auto v = expr.eval_path(i, 0); - if (v.is_null()) { - builder.push_back_null(); - } else { - builder.push_back_opt(v.as_double(), true); - } + return builder.finish(); + } + case common::PrimitiveType::DT_DOUBLE: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_double(), true); } + } - return builder.finish(); - } break; - case common::DataType::BOOLEAN: { - OptionalValueColumnBuilder builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - auto v = expr.eval_path(i, 0); - if (v.is_null()) { - builder.push_back_null(); - } else { - builder.push_back_opt(v.as_bool(), true); - } + return builder.finish(); + } + case common::PrimitiveType::DT_BOOL: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_bool(), true); } + } - return builder.finish(); - } break; - case common::DataType::STRING: { + return builder.finish(); + } + default: + LOG(FATAL) << "not support"; + return nullptr; + } +} + +std::shared_ptr build_optional_column( + const common::IrDataType& data_type, const Expr& expr, size_t row_num) { + switch (data_type.type_case()) { + case common::IrDataType::kDataType: { + switch (data_type.data_type().item_case()) { + case common::DataType::ItemCase::kPrimitiveType: { + return build_optional_primitive_column( + data_type.data_type().primitive_type(), expr, row_num); + } + + case common::DataType::ItemCase::kString: { OptionalValueColumnBuilder builder; builder.reserve(row_num); for (size_t i = 0; i < row_num; ++i) { @@ -218,25 +233,30 @@ std::shared_ptr build_optional_column( } return builder.finish(); - } break; - case common::DataType::TIMESTAMP: { - OptionalValueColumnBuilder builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - auto v = expr.eval_path(i, 0); - if (v.is_null()) { - builder.push_back_null(); - } else { - builder.push_back_opt(v.as_date32(), true); + } + case common::DataType::ItemCase::kTemporal: { + if (data_type.data_type().temporal().item_case() == + common::Temporal::kDate32) { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_date32(), true); + } } - } - return builder.finish(); - } break; + return builder.finish(); + } else { + LOG(FATAL) << "not support type: " + << data_type.data_type().DebugString(); + } + } default: { - LOG(FATAL) << "not support" - << common::DataType_Name(data_type.data_type()); + LOG(FATAL) << "not support" << data_type.data_type().DebugString(); break; } } @@ -252,6 +272,56 @@ std::shared_ptr build_optional_column( return nullptr; } +std::shared_ptr build_primitive_column( + const common::PrimitiveType& data_type, const Expr& expr, size_t row_num) { + switch (data_type) { + case common::PrimitiveType::DT_SIGNED_INT64: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_int64(); + builder.push_back_opt(v); + } + + return builder.finish(); + } + case common::PrimitiveType::DT_SIGNED_INT32: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_int32(); + builder.push_back_opt(v); + } + + return builder.finish(); + } + case common::PrimitiveType::DT_DOUBLE: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_double(); + builder.push_back_opt(v); + } + + return builder.finish(); + } + + case common::PrimitiveType::DT_BOOL: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_bool(); + builder.push_back_opt(v); + } + + return builder.finish(); + } + default: + LOG(FATAL) << "not support"; + return nullptr; + } +} + std::shared_ptr build_column( const common::IrDataType& data_type, const Expr& expr, size_t row_num) { if (expr.is_optional()) { @@ -259,28 +329,12 @@ std::shared_ptr build_column( } switch (data_type.type_case()) { case common::IrDataType::kDataType: { - switch (data_type.data_type()) { - case common::DataType::INT64: { - ValueColumnBuilder builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - auto v = expr.eval_path(i).as_int64(); - builder.push_back_opt(v); - } - - return builder.finish(); - } break; - case common::DataType::INT32: { - ValueColumnBuilder builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - auto v = expr.eval_path(i).as_int32(); - builder.push_back_opt(v); - } - - return builder.finish(); - } break; - case common::DataType::STRING: { + switch (data_type.data_type().item_case()) { + case common::DataType::ItemCase::kPrimitiveType: { + return build_primitive_column(data_type.data_type().primitive_type(), + expr, row_num); + } + case common::DataType::ItemCase::kString: { ValueColumnBuilder builder; builder.reserve(row_num); for (size_t i = 0; i < row_num; ++i) { @@ -289,61 +343,49 @@ std::shared_ptr build_column( } return builder.finish(); - } break; - case common::DataType::DATE32: { - ValueColumnBuilder builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - auto v = expr.eval_path(i).as_date32(); - builder.push_back_opt(v); - } - - return builder.finish(); - } break; - case common::DataType::STRING_ARRAY: { - ValueColumnBuilder> builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - const auto& v = expr.eval_path(i).as_string_set(); - builder.push_back_opt(v); - } + } + case common::DataType::ItemCase::kTemporal: { + if (data_type.data_type().temporal().item_case() == + common::Temporal::kDate32 || + data_type.data_type().temporal().item_case() == + common::Temporal::kTimestamp) { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_date32(); + builder.push_back_opt(v); + } - return builder.finish(); - } break; - case common::DataType::TIMESTAMP: { - ValueColumnBuilder builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - auto v = expr.eval_path(i).as_date32(); - builder.push_back_opt(v); + return builder.finish(); + } else { + LOG(FATAL) << "not support type: " + << data_type.data_type().temporal().DebugString(); } + } - return builder.finish(); - } break; - case common::DataType::BOOLEAN: { - ValueColumnBuilder builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - auto v = expr.eval_path(i).as_bool(); - builder.push_back_opt(v); - } - return builder.finish(); - } break; - case common::DataType::DOUBLE: { - ValueColumnBuilder builder; - builder.reserve(row_num); - for (size_t i = 0; i < row_num; ++i) { - auto v = expr.eval_path(i).as_double(); - builder.push_back_opt(v); + case common::DataType::ItemCase::kArray: { + if (data_type.data_type().array().component_type().item_case() == + common::DataType::ItemCase::kString) { + ValueColumnBuilder> builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + const auto& v = expr.eval_path(i).as_string_set(); + builder.push_back_opt(v); + } + return builder.finish(); + } else { + LOG(FATAL) << "not support: " << data_type.DebugString(); } - return builder.finish(); - } break; + } + case common::DataType::ItemCase::kMap: { + LOG(FATAL) << "not support: " << data_type.DebugString(); + } default: { - LOG(FATAL) << "not support: " - << common::DataType_Name(data_type.data_type()); + LOG(FATAL) << "not support" << data_type.data_type().DebugString(); + break; } } - } break; + case common::IrDataType::kGraphType: { const common::GraphDataType& graph_data_type = data_type.graph_type(); common::GraphDataType_GraphElementOpt elem_opt = @@ -383,17 +425,17 @@ std::shared_ptr build_column( } else { LOG(FATAL) << "unexpected type"; } - } break; + } case common::IrDataType::TYPE_NOT_SET: { return build_column_beta(expr, row_num); - } break; + } default: - LOG(FATAL) << "unexpected type" - << common::DataType_Name(data_type.data_type()); + LOG(FATAL) << "unexpected type" << data_type.data_type().DebugString(); break; } - return nullptr; + return nullptr; + } } std::shared_ptr build_optional_column_beta(const Expr& expr, @@ -563,7 +605,6 @@ std::shared_ptr build_column_beta(const Expr& expr, return nullptr; } - } // namespace runtime } // namespace gs diff --git a/flex/engines/graph_db/runtime/common/rt_any.cc b/flex/engines/graph_db/runtime/common/rt_any.cc index e3cf4a1d55d8..c2f60da57346 100644 --- a/flex/engines/graph_db/runtime/common/rt_any.cc +++ b/flex/engines/graph_db/runtime/common/rt_any.cc @@ -14,6 +14,7 @@ */ #include "flex/engines/graph_db/runtime/common/rt_any.h" +#include "flex/proto_generated_gie/basic_type.pb.h" namespace gs { @@ -53,25 +54,37 @@ RTAnyType parse_from_ir_data_type(const ::common::IrDataType& dt) { switch (dt.type_case()) { case ::common::IrDataType::TypeCase::kDataType: { const ::common::DataType ddt = dt.data_type(); - switch (ddt) { - case ::common::DataType::BOOLEAN: - return RTAnyType::kBoolValue; - case ::common::DataType::INT64: - return RTAnyType::kI64Value; - case ::common::DataType::STRING: + switch (ddt.item_case()) { + case ::common::DataType::kPrimitiveType: { + const ::common::PrimitiveType pt = ddt.primitive_type(); + switch (pt) { + case ::common::PrimitiveType::DT_SIGNED_INT32: + return RTAnyType::kI32Value; + case ::common::PrimitiveType::DT_SIGNED_INT64: + return RTAnyType::kI64Value; + case ::common::PrimitiveType::DT_DOUBLE: + return RTAnyType::kF64Value; + case ::common::PrimitiveType::DT_BOOL: + return RTAnyType::kBoolValue; + default: + LOG(FATAL) << "unrecoginized primitive type - " << pt; + break; + } + } + case ::common::DataType::kString: return RTAnyType::kStringValue; - case ::common::DataType::INT32: - return RTAnyType::kI32Value; - case ::common::DataType::DATE32: - return RTAnyType::kDate32; - case ::common::DataType::STRING_ARRAY: + case ::common::DataType::kTemporal: { + if (ddt.temporal().item_case() == ::common::Temporal::kDate32) { + return RTAnyType::kDate32; + } else { + LOG(FATAL) << "unrecoginized temporal type - " + << ddt.temporal().DebugString(); + } + } + case ::common::DataType::kArray: return RTAnyType::kList; - case ::common::DataType::TIMESTAMP: - return RTAnyType::kDate32; - case ::common::DataType::DOUBLE: - return RTAnyType::kF64Value; default: - LOG(FATAL) << "unrecoginized data type - " << ddt; + LOG(FATAL) << "unrecoginized data type - " << ddt.DebugString(); break; } } break; diff --git a/flex/utils/CMakeLists.txt b/flex/utils/CMakeLists.txt index c25376746fbc..a85c74491710 100644 --- a/flex/utils/CMakeLists.txt +++ b/flex/utils/CMakeLists.txt @@ -32,6 +32,7 @@ set(CODE_GEN_PROTOBUF_FILES ${GIE_COMPILER_PROTO_DIR}/schema.proto ${GIE_COMPILER_PROTO_DIR}/type.proto ${GIE_COMPILER_PROTO_DIR}/stored_procedure.proto + ${GIE_COMPILER_PROTO_DIR}/basic_type.proto ${GIE_COMPILER_PROTO_JOB_DIR}/job_service.proto ) From 9238a28fea42f7b88fd38ebe86e4ff73e6e0ef0f Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 27 Dec 2024 18:42:12 +0800 Subject: [PATCH 08/23] support flex type in physical proto --- .../meta/procedure/StoredProcedureMeta.java | 4 +- .../ir/meta/schema/GSDataTypeConvertor.java | 441 ++++++++--------- .../common/ir/meta/schema/GSDataTypeDesc.java | 16 + .../common/ir/meta/schema/Utils.java | 3 +- .../common/ir/runtime/proto/Utils.java | 62 +-- .../common/ir/runtime/FfiLogicalPlanTest.java | 312 ------------ .../test/resources/ffi_logical_plan_1.json | 453 ------------------ .../test/resources/ffi_logical_plan_2.json | 160 ------- .../test/resources/ffi_logical_plan_3.json | 288 ----------- .../test/resources/ffi_logical_plan_6.json | 204 -------- .../test/resources/ffi_logical_plan_7.json | 258 ---------- .../test/resources/ffi_logical_plan_8.json | 172 ------- .../test/resources/ffi_logical_plan_9.json | 225 --------- .../test/resources/proto/aggregate_test.json | 48 +- .../resources/proto/aggregate_test_2.json | 64 ++- .../test/resources/proto/date_minus_date.json | 42 +- .../proto/date_plus_literal_interval.json | 35 +- .../proto/date_plus_param_interval.json | 43 +- .../test/resources/proto/dedup_test_1.json | 36 +- .../test/resources/proto/dedup_test_2.json | 37 +- .../proto/edge_expand_degree_test.json | 27 +- .../resources/proto/edge_expand_test.json | 19 +- .../proto/edge_expand_vertex_filter_test.json | 50 +- .../proto/edge_expand_vertex_test.json | 19 +- .../edge_expand_vertex_with_filters_test.json | 66 ++- .../proto/expression_with_brace.json | 28 +- .../test/resources/proto/extract_year.json | 20 +- .../src/test/resources/proto/filter_test.json | 40 +- .../test/resources/proto/filter_test_2.json | 43 +- .../src/test/resources/proto/getv_test.json | 34 +- .../proto/getv_with_filter_test.json | 65 ++- .../test/resources/proto/id_in_list_expr.json | 16 +- .../test/resources/proto/intersect_test.json | 114 ++++- .../resources/proto/intersect_test_2.json | 236 +++++++-- .../src/test/resources/proto/join_test_1.json | 44 +- .../src/test/resources/proto/join_test_2.json | 92 +++- .../src/test/resources/proto/limit_test.json | 15 +- .../proto/name_not_containing_expr.json | 29 +- .../proto/partitioned_aggregate_test.json | 48 +- .../proto/partitioned_aggregate_test_2.json | 195 -------- .../proto/partitioned_dedup_test_1.json | 36 +- .../proto/partitioned_dedup_test_2.json | 37 +- .../partitioned_edge_expand_degree_test.json | 27 +- .../proto/partitioned_edge_expand_test.json | 19 +- ...tioned_edge_expand_vertex_filter_test.json | 50 +- .../partitioned_edge_expand_vertex_test.json | 19 +- .../proto/partitioned_filter_test_2.json | 43 +- .../proto/partitioned_getv_test.json | 34 +- .../proto/partitioned_intersect_test.json | 114 ++++- .../proto/partitioned_intersect_test_2.json | 236 +++++++-- .../proto/partitioned_join_test_1.json | 44 +- .../proto/partitioned_join_test_2.json | 92 +++- .../proto/partitioned_project_test.json | 36 +- .../proto/partitioned_project_test_2.json | 52 +- .../proto/partitioned_sort_test.json | 22 +- .../resources/proto/path_expand_test.json | 19 +- .../proto/path_fused_expand_test.json | 19 +- .../test/resources/proto/project_test.json | 36 +- .../test/resources/proto/project_test_2.json | 52 +- .../test/resources/proto/scan_edge_test.json | 4 +- .../resources/proto/scan_filter_test.json | 31 +- .../src/test/resources/proto/scan_test.json | 15 +- .../src/test/resources/proto/sort_test.json | 22 +- .../test/resources/proto/st_path_test.json | 324 ++++++++++--- 64 files changed, 2423 insertions(+), 3063 deletions(-) delete mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java delete mode 100644 interactive_engine/compiler/src/test/resources/ffi_logical_plan_1.json delete mode 100644 interactive_engine/compiler/src/test/resources/ffi_logical_plan_2.json delete mode 100644 interactive_engine/compiler/src/test/resources/ffi_logical_plan_3.json delete mode 100644 interactive_engine/compiler/src/test/resources/ffi_logical_plan_6.json delete mode 100644 interactive_engine/compiler/src/test/resources/ffi_logical_plan_7.json delete mode 100644 interactive_engine/compiler/src/test/resources/ffi_logical_plan_8.json delete mode 100644 interactive_engine/compiler/src/test/resources/ffi_logical_plan_9.json delete mode 100644 interactive_engine/compiler/src/test/resources/proto/partitioned_aggregate_test_2.json diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java index c0e00ddde078..7a4377014e72 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java @@ -172,7 +172,7 @@ public static void perform(StoredProcedureMeta meta, OutputStream outputStream) private static Map createProduceMetaMap(StoredProcedureMeta meta) { GSDataTypeConvertor typeConvertor = - GSDataTypeConvertor.Factory.create(RelDataType.class, typeFactory); + new GSDataTypeConvertor.Calcite(typeFactory); return ImmutableMap.of( Config.NAME.getKey(), meta.name, @@ -217,7 +217,7 @@ private static Map createProduceMetaMap(StoredProcedureMeta meta public static class Deserializer { public static StoredProcedureMeta perform(InputStream inputStream) throws IOException { GSDataTypeConvertor typeConvertor = - GSDataTypeConvertor.Factory.create(RelDataType.class, typeFactory); + new GSDataTypeConvertor.Calcite(typeFactory); Yaml yaml = new Yaml(); Map config = yaml.load(inputStream); return new StoredProcedureMeta( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java index f12a70e8caf8..444add5f15cc 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java @@ -20,6 +20,7 @@ import com.alibaba.graphscope.common.ir.type.ArbitraryArrayType; import com.alibaba.graphscope.common.ir.type.ArbitraryMapType; +import com.alibaba.graphscope.common.ir.type.GraphLabelType; import com.alibaba.graphscope.groot.common.schema.wrapper.DataType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -27,249 +28,231 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.type.IntervalSqlType; import org.apache.calcite.sql.type.SqlTypeName; -import org.checkerframework.checker.nullness.qual.Nullable; import java.util.Map; +/** + * define the convertor from flex type system to the given target type system + * @param + */ public interface GSDataTypeConvertor { + // convert from flex type system to target type system T convert(GSDataTypeDesc from); + // convert from target type system to flex type system GSDataTypeDesc convert(T from); - class Factory { - public static GSDataTypeConvertor create(Class tType, @Nullable Object config) { - if (tType.equals(DataType.class)) { - return new GSDataTypeConvertor() { - @Override - public DataType convert(GSDataTypeDesc from) { - Map typeMap = from.getYamlDesc(); - Object value; - if ((value = typeMap.get("primitive_type")) != null) { - switch (value.toString()) { - case "DT_SIGNED_INT32": - return DataType.INT; - case "DT_SIGNED_INT64": - return DataType.LONG; - case "DT_BOOL": - return DataType.BOOL; - case "DT_FLOAT": - return DataType.FLOAT; - case "DT_DOUBLE": - return DataType.DOUBLE; - default: - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to DataType"); - } - } else if ((value = typeMap.get("string")) != null) { - Map strType = (Map) value; - if (strType.containsKey("long_text")) { - return DataType.STRING; - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to DataType"); - } - } else if ((value = typeMap.get("temporal")) != null) { - Map temporalType = (Map) value; - if (temporalType.containsKey("date32")) { - return DataType.DATE; - } else if (temporalType.containsKey("time32")) { - return DataType.TIME32; - } else if (temporalType.containsKey("timestamp")) { - return DataType.TIMESTAMP; - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to DataType"); - } - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to DataType"); - } - } - - @Override - public GSDataTypeDesc convert(DataType from) { + class Groot implements GSDataTypeConvertor { + @Override + public DataType convert(GSDataTypeDesc from) { + Map typeMap = from.getYamlDesc(); + Object value; + if ((value = typeMap.get("primitive_type")) != null) { + switch (value.toString()) { + case "DT_SIGNED_INT32": + return DataType.INT; + case "DT_SIGNED_INT64": + return DataType.LONG; + case "DT_BOOL": + return DataType.BOOL; + case "DT_FLOAT": + return DataType.FLOAT; + case "DT_DOUBLE": + return DataType.DOUBLE; + default: throw new UnsupportedOperationException( - "convert from DataType to GSDataTypeDesc is unsupported yet"); - } - }; - } else if (tType.equals(RelDataType.class)) { - return new GSDataTypeConvertor() { - @Override - public RelDataType convert(GSDataTypeDesc from) { - RelDataTypeFactory typeFactory = (RelDataTypeFactory) config; - Map typeMap = from.getYamlDesc(); - Object value; - if ((value = typeMap.get("primitive_type")) != null) { - switch (value.toString()) { - case "DT_ANY": - return typeFactory.createSqlType(SqlTypeName.ANY); - case "DT_SIGNED_INT32": - return typeFactory.createSqlType(SqlTypeName.INTEGER); - case "DT_SIGNED_INT64": - return typeFactory.createSqlType(SqlTypeName.BIGINT); - case "DT_BOOL": - return typeFactory.createSqlType(SqlTypeName.BOOLEAN); - case "DT_FLOAT": - return typeFactory.createSqlType(SqlTypeName.FLOAT); - case "DT_DOUBLE": - return typeFactory.createSqlType(SqlTypeName.DOUBLE); - default: - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to RelDataType"); - } - } else if ((value = typeMap.get("string")) != null) { - Map strType = (Map) value; - if (strType.containsKey("long_text")) { - return typeFactory.createSqlType(SqlTypeName.CHAR); - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to RelDataType"); - } - } else if ((value = typeMap.get("temporal")) != null) { - Map temporalType = (Map) value; - if (temporalType.containsKey("date32")) { - return typeFactory.createSqlType(SqlTypeName.DATE); - } else if (temporalType.containsKey("time32")) { - return typeFactory.createSqlType(SqlTypeName.TIME); - } else if (temporalType.containsKey("timestamp")) { - return typeFactory.createSqlType(SqlTypeName.TIMESTAMP); - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" - + from - + "] to RelDataType"); - } - } else if ((value = typeMap.get("array")) != null) { - Map arrayType = (Map) value; - Map componentType = - (Map) arrayType.get("component_type"); - Preconditions.checkArgument( - componentType != null, - "field 'component_type' is required in array type"); - return typeFactory.createArrayType( - convert(new GSDataTypeDesc(componentType)), -1); - } else if ((value = typeMap.get("map")) != null) { - Map mapType = (Map) value; - Map keyType = - (Map) mapType.get("key_type"); - Preconditions.checkArgument( - keyType != null, "field 'key_type' is required in map type"); - Map valueType = - (Map) mapType.get("value_type"); - Preconditions.checkArgument( - valueType != null, - "field 'value_type' is required in map type"); - return typeFactory.createMapType( - convert(new GSDataTypeDesc(keyType)), - convert(new GSDataTypeDesc(valueType))); - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to RelDataType"); - } - } + "can not convert GSDataTypeDesc [" + from + "] to DataType"); + } + } else if ((value = typeMap.get("string")) != null) { + Map strType = (Map) value; + if (strType.containsKey("long_text")) { + return DataType.STRING; + } else { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + from + "] to DataType"); + } + } else if ((value = typeMap.get("temporal")) != null) { + Map temporalType = (Map) value; + if (temporalType.containsKey("date32")) { + return DataType.DATE; + } else if (temporalType.containsKey("time32")) { + return DataType.TIME32; + } else if (temporalType.containsKey("timestamp")) { + return DataType.TIMESTAMP; + } else { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + from + "] to DataType"); + } + } else { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + from + "] to DataType"); + } + } - @Override - public GSDataTypeDesc convert(RelDataType from) { - SqlTypeName typeName = from.getSqlTypeName(); - Map yamlDesc; - switch (typeName) { - case INTEGER: - yamlDesc = ImmutableMap.of("primitive_type", "DT_SIGNED_INT32"); - break; - case BIGINT: - yamlDesc = ImmutableMap.of("primitive_type", "DT_SIGNED_INT64"); - break; - case BOOLEAN: - yamlDesc = ImmutableMap.of("primitive_type", "DT_BOOL"); - break; - case FLOAT: - yamlDesc = ImmutableMap.of("primitive_type", "DT_FLOAT"); - break; - case DOUBLE: - yamlDesc = ImmutableMap.of("primitive_type", "DT_DOUBLE"); - break; - case CHAR: - Map longTextMap = Maps.newHashMap(); - longTextMap.put("long_text", null); - yamlDesc = ImmutableMap.of("string", longTextMap); - break; - case DATE: - Map dateMap = Maps.newHashMap(); - dateMap.put("date32", null); - yamlDesc = ImmutableMap.of("temporal", dateMap); - break; - case TIME: - Map timeMap = Maps.newHashMap(); - timeMap.put("time32", null); - yamlDesc = ImmutableMap.of("temporal", timeMap); - break; - case TIMESTAMP: - Map timestampMap = Maps.newHashMap(); - timestampMap.put("timestamp", null); - yamlDesc = ImmutableMap.of("temporal", timestampMap); - break; - case ARRAY: - case MULTISET: - Map componentType; - if (from instanceof ArbitraryArrayType) { - componentType = ImmutableMap.of("primitive_type", "DT_ANY"); - } else { - componentType = convert(from.getComponentType()).getYamlDesc(); - } - yamlDesc = - ImmutableMap.of( - "array", - ImmutableMap.of( - "component_type", - componentType, - "max_length", - Integer.MAX_VALUE)); - break; - case MAP: - Map keyType; - Map valueType; - if (from instanceof ArbitraryMapType) { - keyType = ImmutableMap.of("primitive_type", "DT_ANY"); - valueType = ImmutableMap.of("primitive_type", "DT_ANY"); - } else { - keyType = convert(from.getKeyType()).getYamlDesc(); - valueType = convert(from.getValueType()).getYamlDesc(); - } - yamlDesc = - ImmutableMap.of( - "map", - ImmutableMap.of( - "key_type", - keyType, - "value_type", - valueType)); - break; - default: - throw new UnsupportedOperationException( - "can not convert RelDataType [" - + from - + "] to GSDataTypeDesc"); - } - return new GSDataTypeDesc(yamlDesc); - } - }; + @Override + public GSDataTypeDesc convert(DataType from) { + throw new UnsupportedOperationException( + "convert from DataType to GSDataTypeDesc is unsupported yet"); + } + } + + class Calcite implements GSDataTypeConvertor { + private final RelDataTypeFactory typeFactory; + + public Calcite(RelDataTypeFactory typeFactory) { + this.typeFactory = typeFactory; + } + + @Override + public RelDataType convert(GSDataTypeDesc from) { + Map typeMap = from.getYamlDesc(); + Object value; + if ((value = typeMap.get("primitive_type")) != null) { + switch (value.toString()) { + case "DT_ANY": + return typeFactory.createSqlType(SqlTypeName.ANY); + case "DT_SIGNED_INT32": + return typeFactory.createSqlType(SqlTypeName.INTEGER); + case "DT_SIGNED_INT64": + return typeFactory.createSqlType(SqlTypeName.BIGINT); + case "DT_BOOL": + return typeFactory.createSqlType(SqlTypeName.BOOLEAN); + case "DT_FLOAT": + return typeFactory.createSqlType(SqlTypeName.FLOAT); + case "DT_DOUBLE": + return typeFactory.createSqlType(SqlTypeName.DOUBLE); + default: + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + from + "] to RelDataType"); + } + } else if ((value = typeMap.get("string")) != null) { + Map strType = (Map) value; + if (strType.containsKey("long_text")) { + return typeFactory.createSqlType(SqlTypeName.CHAR); + } else { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + from + "] to RelDataType"); + } + } else if ((value = typeMap.get("temporal")) != null) { + Map temporalType = (Map) value; + if (temporalType.containsKey("date32")) { + return typeFactory.createSqlType(SqlTypeName.DATE); + } else if (temporalType.containsKey("time32")) { + return typeFactory.createSqlType(SqlTypeName.TIME); + } else if (temporalType.containsKey("timestamp")) { + return typeFactory.createSqlType(SqlTypeName.TIMESTAMP); + } else { + throw new UnsupportedOperationException( + "can not convert GSDataTypeDesc [" + from + "] to RelDataType"); + } + } else if ((value = typeMap.get("array")) != null) { + Map arrayType = (Map) value; + Map componentType = + (Map) arrayType.get("component_type"); + Preconditions.checkArgument( + componentType != null, "field 'component_type' is required in array type"); + return typeFactory.createArrayType(convert(new GSDataTypeDesc(componentType)), -1); + } else if ((value = typeMap.get("map")) != null) { + Map mapType = (Map) value; + Map keyType = (Map) mapType.get("key_type"); + Preconditions.checkArgument( + keyType != null, "field 'key_type' is required in map type"); + Map valueType = (Map) mapType.get("value_type"); + Preconditions.checkArgument( + valueType != null, "field 'value_type' is required in map type"); + return typeFactory.createMapType( + convert(new GSDataTypeDesc(keyType)), + convert(new GSDataTypeDesc(valueType))); } else { throw new UnsupportedOperationException( - "unsupported type class " - + tType - + " in GSDataTypeConvertor.Factory.create"); + "can not convert GSDataTypeDesc [" + from + "] to RelDataType"); + } + } + + @Override + public GSDataTypeDesc convert(RelDataType from) { + if (from instanceof IntervalSqlType) { + return new GSDataTypeDesc(ImmutableMap.of("primitive_type", "DT_SIGNED_INT64")); + } else if (from instanceof GraphLabelType) { + return new GSDataTypeDesc(ImmutableMap.of("primitive_type", "DT_SIGNED_INT32")); + } + SqlTypeName typeName = from.getSqlTypeName(); + Map yamlDesc; + switch (typeName) { + case INTEGER: + yamlDesc = ImmutableMap.of("primitive_type", "DT_SIGNED_INT32"); + break; + case BIGINT: + yamlDesc = ImmutableMap.of("primitive_type", "DT_SIGNED_INT64"); + break; + case BOOLEAN: + yamlDesc = ImmutableMap.of("primitive_type", "DT_BOOL"); + break; + case FLOAT: + yamlDesc = ImmutableMap.of("primitive_type", "DT_FLOAT"); + break; + case DOUBLE: + yamlDesc = ImmutableMap.of("primitive_type", "DT_DOUBLE"); + break; + case CHAR: + Map longTextMap = Maps.newHashMap(); + longTextMap.put("long_text", ImmutableMap.of()); + yamlDesc = ImmutableMap.of("string", longTextMap); + break; + case DATE: + Map dateMap = Maps.newHashMap(); + dateMap.put("date32", ImmutableMap.of()); + yamlDesc = ImmutableMap.of("temporal", dateMap); + break; + case TIME: + Map timeMap = Maps.newHashMap(); + timeMap.put("time32", ImmutableMap.of()); + yamlDesc = ImmutableMap.of("temporal", timeMap); + break; + case TIMESTAMP: + Map timestampMap = Maps.newHashMap(); + timestampMap.put("timestamp", ImmutableMap.of()); + yamlDesc = ImmutableMap.of("temporal", timestampMap); + break; + case ARRAY: + case MULTISET: + Map componentType; + if (from instanceof ArbitraryArrayType) { + componentType = ImmutableMap.of("primitive_type", "DT_ANY"); + } else { + componentType = convert(from.getComponentType()).getYamlDesc(); + } + yamlDesc = + ImmutableMap.of( + "array", + ImmutableMap.of( + "component_type", + componentType, + "max_length", + Integer.MAX_VALUE)); + break; + case MAP: + Map keyType; + Map valueType; + if (from instanceof ArbitraryMapType) { + keyType = ImmutableMap.of("primitive_type", "DT_ANY"); + valueType = ImmutableMap.of("primitive_type", "DT_ANY"); + } else { + keyType = convert(from.getKeyType()).getYamlDesc(); + valueType = convert(from.getValueType()).getYamlDesc(); + } + yamlDesc = + ImmutableMap.of( + "map", + ImmutableMap.of("key_type", keyType, "value_type", valueType)); + break; + default: + throw new UnsupportedOperationException( + "can not convert RelDataType [" + from + "] to GSDataTypeDesc"); } + return new GSDataTypeDesc(yamlDesc); } } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeDesc.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeDesc.java index 9f280491e34b..7d42b29205b4 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeDesc.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeDesc.java @@ -18,12 +18,19 @@ package com.alibaba.graphscope.common.ir.meta.schema; +import com.alibaba.graphscope.proto.type.Common; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.util.JsonFormat; + import java.util.Map; public class GSDataTypeDesc { // support more format of GSDataTypeDesc, i.e. JSON, proto, etc. private final Map yamlDesc; + // flex type in proto format + private Common.DataType protoDesc; + public GSDataTypeDesc(Map yamlDesc) { this.yamlDesc = yamlDesc; } @@ -32,6 +39,15 @@ public Map getYamlDesc() { return yamlDesc; } + public Common.DataType getProtoDesc() throws Exception { + if (protoDesc != null) return protoDesc; + Common.DataType.Builder protoBuilder = Common.DataType.newBuilder(); + String jsonDesc = new ObjectMapper().writeValueAsString(yamlDesc); + JsonFormat.parser().merge(jsonDesc, protoBuilder); + protoDesc = protoBuilder.build(); + return protoDesc; + } + public String toString() { return yamlDesc.toString(); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java index 0b59ccf7d216..0d1bffe79161 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java @@ -51,8 +51,7 @@ public static final GraphSchema buildSchemaFromYaml(String schemaYaml) { Map vertexMap = Maps.newHashMap(); Map edgeMap = Maps.newHashMap(); Map propNameToIdMap = Maps.newHashMap(); - GSDataTypeConvertor typeConvertor = - GSDataTypeConvertor.Factory.create(DataType.class, null); + GSDataTypeConvertor typeConvertor = new GSDataTypeConvertor.Groot(); builderGraphElementFromYaml( (List) Objects.requireNonNull( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java index a11f2eae2c59..3dd0f7a83675 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java @@ -16,15 +16,12 @@ package com.alibaba.graphscope.common.ir.runtime.proto; +import com.alibaba.graphscope.common.ir.meta.schema.GSDataTypeConvertor; import com.alibaba.graphscope.common.ir.rel.type.group.GraphAggCall; import com.alibaba.graphscope.common.ir.rex.RexVariableAliasCollector; import com.alibaba.graphscope.common.ir.tools.AliasInference; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; -import com.alibaba.graphscope.common.ir.type.GraphLabelType; -import com.alibaba.graphscope.common.ir.type.GraphNameOrId; -import com.alibaba.graphscope.common.ir.type.GraphPathType; -import com.alibaba.graphscope.common.ir.type.GraphProperty; -import com.alibaba.graphscope.common.ir.type.GraphSchemaType; +import com.alibaba.graphscope.common.ir.type.*; import com.alibaba.graphscope.gaia.proto.*; import com.alibaba.graphscope.gaia.proto.GraphAlgebra.GroupBy.AggFunc.Aggregate; import com.google.common.base.Preconditions; @@ -274,54 +271,13 @@ public static final OuterExpression.ExprOpr protoOperator(SqlOperator operator) } } - public static final Common.DataType protoBasicDataType(RelDataType basicType) { - // hack ways: convert interval type to int64 to avoid complexity - if (basicType instanceof IntervalSqlType) return Common.DataType.INT64; - if (basicType instanceof GraphLabelType) return Common.DataType.INT32; - switch (basicType.getSqlTypeName()) { - case NULL: - return Common.DataType.NONE; - case BOOLEAN: - return Common.DataType.BOOLEAN; - case INTEGER: - return Common.DataType.INT32; - case BIGINT: - return Common.DataType.INT64; - case CHAR: - return Common.DataType.STRING; - case DECIMAL: - case FLOAT: - case DOUBLE: - return Common.DataType.DOUBLE; - case MULTISET: - case ARRAY: - RelDataType elementType = basicType.getComponentType(); - switch (elementType.getSqlTypeName()) { - case INTEGER: - return Common.DataType.INT32_ARRAY; - case BIGINT: - return Common.DataType.INT64_ARRAY; - case CHAR: - return Common.DataType.STRING_ARRAY; - case DECIMAL: - case FLOAT: - case DOUBLE: - return Common.DataType.DOUBLE_ARRAY; - default: - throw new UnsupportedOperationException( - "array of element type " - + elementType.getSqlTypeName() - + " is unsupported yet"); - } - case DATE: - return Common.DataType.DATE32; - case TIME: - return Common.DataType.TIME32; - case TIMESTAMP: - return Common.DataType.TIMESTAMP; - default: - throw new UnsupportedOperationException( - "basic type " + basicType.getSqlTypeName() + " is unsupported yet"); + public static final com.alibaba.graphscope.proto.type.Common.DataType protoBasicDataType( + RelDataType basicType) { + try { + GSDataTypeConvertor convertor = new GSDataTypeConvertor.Calcite(null); + return convertor.convert(basicType).getProtoDesc(); + } catch (Exception e) { + throw new RuntimeException(e); } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java deleted file mode 100644 index dcb3decb3899..000000000000 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.ir.runtime; - -import com.alibaba.graphscope.common.config.Configs; -import com.alibaba.graphscope.common.ir.Utils; -import com.alibaba.graphscope.common.ir.runtime.ffi.FfiPhysicalBuilder; -import com.alibaba.graphscope.common.ir.tools.GraphBuilder; -import com.alibaba.graphscope.common.ir.tools.GraphRexBuilder; -import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; -import com.alibaba.graphscope.common.ir.tools.LogicalPlan; -import com.alibaba.graphscope.common.ir.tools.config.*; -import com.alibaba.graphscope.common.ir.type.GraphProperty; -import com.alibaba.graphscope.common.utils.FileUtils; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; - -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.junit.Assert; -import org.junit.Test; - -public class FfiLogicalPlanTest { - // Match (x:person)-[:knows*1..3]->(:person {age: 10}) - // Return count(*) - @Test - public void logical_plan_1_test() throws Exception { - GraphBuilder builder = - Utils.mockGraphBuilder() - .source( - new SourceConfig( - GraphOpt.Source.VERTEX, - new LabelConfig(false).addLabel("person"))); - PathExpandConfig.Builder pxdBuilder = PathExpandConfig.newBuilder(builder); - GetVConfig getVConfig = - new GetVConfig(GraphOpt.GetV.END, new LabelConfig(false).addLabel("person")); - PathExpandConfig pxdConfig = - pxdBuilder - .expand( - new ExpandConfig( - GraphOpt.Expand.OUT, - new LabelConfig(false).addLabel("knows"))) - .getV(getVConfig) - .range(1, 3) - .pathOpt(GraphOpt.PathExpandPath.SIMPLE) - .resultOpt(GraphOpt.PathExpandResult.ALL_V) - .buildConfig(); - RelNode aggregate = - builder.source( - new SourceConfig( - GraphOpt.Source.VERTEX, - new LabelConfig(false).addLabel("person"), - "x")) - .pathExpand(pxdConfig) - .getV(getVConfig) - .filter( - builder.call( - GraphStdOperatorTable.EQUALS, - pxdBuilder.variable(null, "age"), - pxdBuilder.literal(10))) - .aggregate(builder.groupKey(), builder.count(builder.variable("x"))) - .build(); - Assert.assertEquals( - "GraphLogicalAggregate(keys=[{variables=[], aliases=[]}], values=[[{operands=[x]," - + " aggFunction=COUNT, alias='$f0', distinct=false}]])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}], alias=[_]," - + " fusedFilter=[[=(_.age, 10)]], opt=[END])\n" - + " GraphLogicalPathExpand(expand=[GraphLogicalExpand(tableConfig=[{isAll=false," - + " tables=[knows]}], alias=[_], opt=[OUT])\n" - + "], getV=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}]," - + " alias=[_], opt=[END])\n" - + "], offset=[1], fetch=[3], path_opt=[SIMPLE], result_opt=[ALL_V]," - + " alias=[_])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," - + " alias=[x], opt=[VERTEX])", - aggregate.explain().trim()); - try (PhysicalBuilder ffiBuilder = - new FfiPhysicalBuilder( - getMockGraphConfig(), Utils.schemaMeta, new LogicalPlan(aggregate))) { - PhysicalPlan plan = ffiBuilder.build(); - Assert.assertEquals( - FileUtils.readJsonFromResource("ffi_logical_plan_1.json"), plan.explain()); - } - } - - // Match (x:person) where x.age = $age - @Test - public void logical_plan_2_test() throws Exception { - GraphBuilder builder = Utils.mockGraphBuilder(); - RelNode filter = - builder.source( - new SourceConfig( - GraphOpt.Source.VERTEX, - new LabelConfig(false).addLabel("person"), - "x")) - .filter( - builder.call( - GraphStdOperatorTable.EQUALS, - builder.variable("x", "age"), - ((GraphRexBuilder) builder.getRexBuilder()) - .makeGraphDynamicParam("age", 0))) - .build(); - Assert.assertEquals( - "GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}], alias=[x]," - + " fusedFilter=[[=(_.age, ?0)]], opt=[VERTEX])", - filter.explain().trim()); - try (PhysicalBuilder ffiBuilder = - new FfiPhysicalBuilder( - getMockGraphConfig(), Utils.schemaMeta, new LogicalPlan(filter))) { - PhysicalPlan plan = ffiBuilder.build(); - Assert.assertEquals( - FileUtils.readJsonFromResource("ffi_logical_plan_2.json"), plan.explain()); - } - } - - // Match (n) Return distinct n; - @Test - public void logical_plan_3_test() throws Exception { - GraphBuilder builder = Utils.mockGraphBuilder(); - RelNode project = - builder.source( - new SourceConfig( - GraphOpt.Source.VERTEX, - new LabelConfig(false).addLabel("person"), - "n")) - .aggregate(builder.groupKey(builder.variable("n"))) - .build(); - Assert.assertEquals( - "GraphLogicalAggregate(keys=[{variables=[n], aliases=[n]}], values=[[]])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," - + " alias=[n], opt=[VERTEX])", - project.explain().trim()); - try (PhysicalBuilder ffiBuilder = - new FfiPhysicalBuilder( - getMockGraphConfig(), Utils.schemaMeta, new LogicalPlan(project))) { - PhysicalPlan plan = ffiBuilder.build(); - Assert.assertEquals( - FileUtils.readJsonFromResource("ffi_logical_plan_3.json"), plan.explain()); - } - } - - @Test - public void logical_plan_4_test() throws Exception { - LogicalPlan logicalPlan = - com.alibaba.graphscope.cypher.antlr4.Utils.evalLogicalPlan( - "Call ldbc_ic2(10l, 20120112l)", "config/modern/graph.yaml"); - try (PhysicalBuilder ffiBuilder = - new ProcedurePhysicalBuilder(Utils.configs, Utils.schemaMeta, logicalPlan)) { - PhysicalPlan plan = ffiBuilder.build(); - Assert.assertEquals( - FileUtils.readJsonFromResource("call_procedure.json"), plan.explain()); - } - } - - // Match (a:person) Return case when a.name = 'marko' then 1 else 3 end as case - @Test - public void logical_plan_5_test() throws Exception { - GraphBuilder builder = - Utils.mockGraphBuilder() - .source( - new SourceConfig( - GraphOpt.Source.VERTEX, - new LabelConfig(false).addLabel("person"), - "a")); - RexNode caseExpr = - builder.call( - GraphStdOperatorTable.CASE, - builder.call( - GraphStdOperatorTable.EQUALS, - builder.variable("a", "name"), - builder.literal("marko")), - builder.literal(1), - builder.literal(3)); - RelNode project = - builder.project(ImmutableList.of(caseExpr), ImmutableList.of("case")).build(); - Assert.assertEquals( - "GraphLogicalProject(case=[CASE(=(a.name, _UTF-8'marko'), 1, 3)]," - + " isAppend=[false])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," - + " alias=[a], opt=[VERTEX])", - project.explain().trim()); - try (PhysicalBuilder ffiBuilder = - new FfiPhysicalBuilder( - getMockGraphConfig(), Utils.schemaMeta, new LogicalPlan(project))) { - PhysicalPlan plan = ffiBuilder.build(); - Assert.assertEquals(FileUtils.readJsonFromResource("case_when.json"), plan.explain()); - } - } - - // test conversion from search operator to within operator in ir core - // a.age SEARCH [1, 2, 3] -> a.age within [1, 2, 3] - @Test - public void logical_plan_6_test() throws Exception { - GraphBuilder builder = Utils.mockGraphBuilder(); - RexBuilder rexBuilder = builder.getRexBuilder(); - RelNode node = - builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) - .filter( - rexBuilder.makeIn( - builder.variable(null, "age"), - ImmutableList.of( - builder.literal(1), - builder.literal(2), - builder.literal(3)))) - .build(); - Assert.assertEquals( - "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," - + " alias=[_], fusedFilter=[[SEARCH(_.age, Sarg[1, 2, 3])]]," - + " opt=[VERTEX])", - node.explain().trim()); - try (PhysicalBuilder ffiBuilder = - new FfiPhysicalBuilder( - getMockGraphConfig(), Utils.schemaMeta, new LogicalPlan(node))) { - PhysicalPlan physicalPlan = ffiBuilder.build(); - Assert.assertEquals( - FileUtils.readJsonFromResource("ffi_logical_plan_6.json"), - physicalPlan.explain()); - } - } - - // test conversion from search continuous ranges to compositions of 'and' in ir core - // a.age SEARCH [[1..10]] -> a.age >= 1 and a.age <= 10 - @Test - public void logical_plan_7_test() throws Exception { - GraphBuilder builder = Utils.mockGraphBuilder(); - RexBuilder rexBuilder = builder.getRexBuilder(); - RelNode node = - builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) - .filter( - rexBuilder.makeBetween( - builder.variable(null, "age"), - builder.literal(1), - builder.literal(10))) - .build(); - Assert.assertEquals( - "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," - + " alias=[_], fusedFilter=[[SEARCH(_.age, Sarg[[1..10]])]]," - + " opt=[VERTEX])", - node.explain().trim()); - try (PhysicalBuilder ffiBuilder = - new FfiPhysicalBuilder( - getMockGraphConfig(), Utils.schemaMeta, new LogicalPlan(node))) { - PhysicalPlan physicalPlan = ffiBuilder.build(); - Assert.assertEquals( - FileUtils.readJsonFromResource("ffi_logical_plan_7.json"), - physicalPlan.explain()); - } - } - - // test conversion of index predicate - // ~id SEARCH [1, 2] -> ~id == 1 or ~id == 2 - @Test - public void logical_plan_8_test() throws Exception { - GraphBuilder builder = Utils.mockGraphBuilder(); - RexBuilder rexBuilder = builder.getRexBuilder(); - RelNode node = - builder.source(new SourceConfig(GraphOpt.Source.VERTEX)) - .filter( - rexBuilder.makeIn( - builder.variable(null, GraphProperty.ID_KEY), - ImmutableList.of(builder.literal(1), builder.literal(2)))) - .build(); - Assert.assertEquals( - "GraphLogicalSource(tableConfig=[{isAll=true, tables=[software, person]}]," - + " alias=[_], opt=[VERTEX], uniqueKeyFilters=[SEARCH(_.~id," - + " Sarg[1, 2])])", - node.explain().trim()); - try (PhysicalBuilder ffiBuilder = - new FfiPhysicalBuilder( - getMockGraphConfig(), Utils.schemaMeta, new LogicalPlan(node))) { - PhysicalPlan physicalPlan = ffiBuilder.build(); - Assert.assertEquals( - FileUtils.readJsonFromResource("ffi_logical_plan_8.json"), - physicalPlan.explain()); - } - } - - @Test - public void logical_plan_9_test() throws Exception { - RelNode node = - com.alibaba.graphscope.cypher.antlr4.Utils.eval( - "Match (a) Return count(distinct a.name, a.age)") - .build(); - try (PhysicalBuilder ffiBuilder = - new FfiPhysicalBuilder( - getMockGraphConfig(), Utils.schemaMeta, new LogicalPlan(node))) { - PhysicalPlan physicalPlan = ffiBuilder.build(); - Assert.assertEquals( - FileUtils.readJsonFromResource("ffi_logical_plan_9.json"), - physicalPlan.explain()); - } - } - - private Configs getMockGraphConfig() { - return new Configs(ImmutableMap.of("servers", "1", "workers", "1")); - } -} diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_1.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_1.json deleted file mode 100644 index c3544a40c16a..000000000000 --- a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_1.json +++ /dev/null @@ -1,453 +0,0 @@ -{ - "nodes": [ - { - "opr": { - "opr": { - "Scan": { - "scan_opt": 0, - "alias": { - "item": { - "Id": 0 - } - }, - "params": { - "tables": [ - { - "item": { - "Id": 0 - } - } - ], - "columns": [], - "is_all_columns": false, - "limit": null, - "predicate": null, - "sample_ratio": 1.0, - "extra": {} - }, - "idx_predicate": null, - "is_count_only": false, - "meta_data": { - "type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - }, - "alias": 0 - } - } - } - }, - "children": [ - 1 - ] - }, - { - "opr": { - "opr": { - "Path": { - "base": { - "edge_expand": { - "v_tag": null, - "direction": 0, - "params": { - "tables": [ - { - "item": { - "Id": 0 - } - } - ], - "columns": [], - "is_all_columns": false, - "limit": null, - "predicate": null, - "sample_ratio": 1.0, - "extra": {} - }, - "alias": null, - "expand_opt": 1, - "meta_data": { - "type": { - "type": { - "GraphType": { - "element_opt": 1, - "graph_data_type": [ - { - "label": { - "label": 0, - "src_label": 0, - "dst_label": 0 - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "weight" - } - }, - "type": 3 - } - ] - } - ] - } - } - }, - "alias": -1 - }, - "is_optional": false - }, - "get_v": { - "tag": null, - "opt": 1, - "params": { - "tables": [ - { - "item": { - "Id": 0 - } - } - ], - "columns": [], - "is_all_columns": false, - "limit": null, - "predicate": null, - "sample_ratio": 1.0, - "extra": {} - }, - "alias": null, - "meta_data": { - "type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - }, - "alias": -1 - } - } - }, - "start_tag": null, - "alias": null, - "hop_range": { - "lower": 1, - "upper": 4 - }, - "path_opt": 1, - "result_opt": 1, - "condition": null, - "is_optional": false - } - } - }, - "children": [ - 2 - ] - }, - { - "opr": { - "opr": { - "Vertex": { - "tag": null, - "opt": 1, - "params": { - "tables": [ - { - "item": { - "Id": 0 - } - } - ], - "columns": [], - "is_all_columns": false, - "limit": null, - "predicate": { - "operators": [ - { - "node_type": { - "type": { - "DataType": 1 - } - }, - "item": { - "Var": { - "tag": null, - "property": { - "item": { - "Key": { - "item": { - "Name": "age" - } - } - } - }, - "node_type": { - "type": { - "DataType": 1 - } - } - } - } - }, - { - "node_type": { - "type": { - "DataType": 0 - } - }, - "item": { - "Logical": 0 - } - }, - { - "node_type": { - "type": { - "DataType": 1 - } - }, - "item": { - "Const": { - "item": { - "I32": 10 - } - } - } - } - ] - }, - "sample_ratio": 1.0, - "extra": {} - }, - "alias": null, - "meta_data": { - "type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - }, - "alias": -1 - } - } - } - }, - "children": [ - 3 - ] - }, - { - "opr": { - "opr": { - "GroupBy": { - "mappings": [], - "functions": [ - { - "vars": [ - { - "tag": { - "item": { - "Id": 0 - } - }, - "property": null, - "node_type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - } - } - ], - "aggregate": 3, - "alias": { - "item": { - "Id": 1 - } - } - } - ], - "meta_data": [ - { - "type": { - "type": { - "DataType": 2 - } - }, - "alias": 1 - } - ] - } - } - }, - "children": [ - 4 - ] - }, - { - "opr": { - "opr": { - "Sink": { - "tags": [], - "sink_target": { - "inner": { - "SinkDefault": { - "id_name_mappings": [] - } - } - } - } - } - }, - "children": [] - } - ], - "roots": [ - 0 - ] -} diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_2.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_2.json deleted file mode 100644 index 22ebbe6d70cc..000000000000 --- a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_2.json +++ /dev/null @@ -1,160 +0,0 @@ -{ - "nodes": [ - { - "opr": { - "opr": { - "Scan": { - "scan_opt": 0, - "alias": { - "item": { - "Id": 0 - } - }, - "params": { - "tables": [ - { - "item": { - "Id": 0 - } - } - ], - "columns": [], - "is_all_columns": false, - "limit": null, - "predicate": { - "operators": [ - { - "node_type": { - "type": { - "DataType": 1 - } - }, - "item": { - "Var": { - "tag": null, - "property": { - "item": { - "Key": { - "item": { - "Name": "age" - } - } - } - }, - "node_type": { - "type": { - "DataType": 1 - } - } - } - } - }, - { - "node_type": { - "type": { - "DataType": 0 - } - }, - "item": { - "Logical": 0 - } - }, - { - "node_type": { - "type": { - "DataType": 1 - } - }, - "item": { - "Param": { - "name": "age", - "index": 0, - "data_type": { - "type": { - "DataType": 1 - } - } - } - } - } - ] - }, - "sample_ratio": 1.0, - "extra": {} - }, - "idx_predicate": null, - "is_count_only": false, - "meta_data": { - "type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - }, - "alias": 0 - } - } - } - }, - "children": [ - 1 - ] - }, - { - "opr": { - "opr": { - "Sink": { - "tags": [], - "sink_target": { - "inner": { - "SinkDefault": { - "id_name_mappings": [] - } - } - } - } - } - }, - "children": [] - } - ], - "roots": [ - 0 - ] -} diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_3.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_3.json deleted file mode 100644 index d61487a842a5..000000000000 --- a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_3.json +++ /dev/null @@ -1,288 +0,0 @@ -{ - "nodes": [ - { - "opr": { - "opr": { - "Scan": { - "scan_opt": 0, - "alias": { - "item": { - "Id": 0 - } - }, - "params": { - "tables": [ - { - "item": { - "Id": 0 - } - } - ], - "columns": [], - "is_all_columns": false, - "limit": null, - "predicate": null, - "sample_ratio": 1.0, - "extra": {} - }, - "idx_predicate": null, - "is_count_only": false, - "meta_data": { - "type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - }, - "alias": 0 - } - } - } - }, - "children": [ - 1 - ] - }, - { - "opr": { - "opr": { - "Project": { - "mappings": [ - { - "expr": { - "operators": [ - { - "node_type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - }, - "item": { - "Var": { - "tag": { - "item": { - "Id": 0 - } - }, - "property": null, - "node_type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - } - } - } - } - ] - }, - "alias": { - "item": { - "Id": 0 - } - } - } - ], - "is_append": false, - "meta_data": [] - } - } - }, - "children": [ - 2 - ] - }, - { - "opr": { - "opr": { - "Dedup": { - "keys": [ - { - "tag": { - "item": { - "Id": 0 - } - }, - "property": null, - "node_type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - } - } - ] - } - } - }, - "children": [ - 3 - ] - }, - { - "opr": { - "opr": { - "Sink": { - "tags": [], - "sink_target": { - "inner": { - "SinkDefault": { - "id_name_mappings": [] - } - } - } - } - } - }, - "children": [] - } - ], - "roots": [ - 0 - ] -} diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_6.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_6.json deleted file mode 100644 index 312cb941531e..000000000000 --- a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_6.json +++ /dev/null @@ -1,204 +0,0 @@ -{ - "nodes": [ - { - "opr": { - "opr": { - "Scan": { - "scan_opt": 0, - "alias": null, - "params": { - "tables": [ - { - "item": { - "Id": 0 - } - }, - { - "item": { - "Id": 1 - } - } - ], - "columns": [], - "is_all_columns": false, - "limit": null, - "predicate": { - "operators": [ - { - "node_type": { - "type": { - "DataType": 1 - } - }, - "item": { - "Var": { - "tag": null, - "property": { - "item": { - "Key": { - "item": { - "Name": "age" - } - } - } - }, - "node_type": { - "type": { - "DataType": 1 - } - } - } - } - }, - { - "node_type": { - "type": { - "DataType": 0 - } - }, - "item": { - "Logical": 6 - } - }, - { - "node_type": { - "type": { - "DataType": 1 - } - }, - "item": { - "Const": { - "item": { - "I32Array": { - "item": [ - 1, - 2, - 3 - ] - } - } - } - } - } - ] - }, - "sample_ratio": 1.0, - "extra": {} - }, - "idx_predicate": null, - "is_count_only": false, - "meta_data": { - "type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 1, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "lang" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "creationDate" - } - }, - "type": 12 - } - ] - }, - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - }, - "alias": -1 - } - } - } - }, - "children": [ - 1 - ] - }, - { - "opr": { - "opr": { - "Sink": { - "tags": [], - "sink_target": { - "inner": { - "SinkDefault": { - "id_name_mappings": [] - } - } - } - } - } - }, - "children": [] - } - ], - "roots": [ - 0 - ] -} diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_7.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_7.json deleted file mode 100644 index f83d67341255..000000000000 --- a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_7.json +++ /dev/null @@ -1,258 +0,0 @@ -{ - "nodes": [ - { - "opr": { - "opr": { - "Scan": { - "scan_opt": 0, - "alias": null, - "params": { - "tables": [ - { - "item": { - "Id": 0 - } - }, - { - "item": { - "Id": 1 - } - } - ], - "columns": [], - "is_all_columns": false, - "limit": null, - "predicate": { - "operators": [ - { - "node_type": { - "type": { - "DataType": 1 - } - }, - "item": { - "Var": { - "tag": null, - "property": { - "item": { - "Key": { - "item": { - "Name": "age" - } - } - } - }, - "node_type": { - "type": { - "DataType": 1 - } - } - } - } - }, - { - "node_type": { - "type": { - "DataType": 0 - } - }, - "item": { - "Logical": 5 - } - }, - { - "node_type": { - "type": { - "DataType": 1 - } - }, - "item": { - "Const": { - "item": { - "I32": 1 - } - } - } - }, - { - "node_type": { - "type": { - "DataType": 0 - } - }, - "item": { - "Logical": 10 - } - }, - { - "node_type": { - "type": { - "DataType": 1 - } - }, - "item": { - "Var": { - "tag": null, - "property": { - "item": { - "Key": { - "item": { - "Name": "age" - } - } - } - }, - "node_type": { - "type": { - "DataType": 1 - } - } - } - } - }, - { - "node_type": { - "type": { - "DataType": 0 - } - }, - "item": { - "Logical": 3 - } - }, - { - "node_type": { - "type": { - "DataType": 1 - } - }, - "item": { - "Const": { - "item": { - "I32": 10 - } - } - } - } - ] - }, - "sample_ratio": 1.0, - "extra": {} - }, - "idx_predicate": null, - "is_count_only": false, - "meta_data": { - "type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 1, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "lang" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "creationDate" - } - }, - "type": 12 - } - ] - }, - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - }, - "alias": -1 - } - } - } - }, - "children": [ - 1 - ] - }, - { - "opr": { - "opr": { - "Sink": { - "tags": [], - "sink_target": { - "inner": { - "SinkDefault": { - "id_name_mappings": [] - } - } - } - } - } - }, - "children": [] - } - ], - "roots": [ - 0 - ] -} diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_8.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_8.json deleted file mode 100644 index 6b1ecf9887fd..000000000000 --- a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_8.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "nodes": [ - { - "opr": { - "opr": { - "Scan": { - "scan_opt": 0, - "alias": null, - "params": { - "tables": [ - { - "item": { - "Id": 0 - } - }, - { - "item": { - "Id": 1 - } - } - ], - "columns": [], - "is_all_columns": false, - "limit": null, - "predicate": null, - "sample_ratio": 1.0, - "extra": {} - }, - "idx_predicate": { - "or_predicates": [ - { - "predicates": [ - { - "key": { - "item": { - "Id": {} - } - }, - "cmp": 6, - "value": { - "Const": { - "item": { - "I32Array": { - "item": [ - 1, - 2 - ] - } - } - } - } - } - ] - } - ] - }, - "is_count_only": false, - "meta_data": { - "type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 1, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "lang" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "creationDate" - } - }, - "type": 12 - } - ] - }, - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - }, - "alias": -1 - } - } - } - }, - "children": [ - 1 - ] - }, - { - "opr": { - "opr": { - "Sink": { - "tags": [], - "sink_target": { - "inner": { - "SinkDefault": { - "id_name_mappings": [] - } - } - } - } - } - }, - "children": [] - } - ], - "roots": [ - 0 - ] -} diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_9.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan_9.json deleted file mode 100644 index 9f7927954fb4..000000000000 --- a/interactive_engine/compiler/src/test/resources/ffi_logical_plan_9.json +++ /dev/null @@ -1,225 +0,0 @@ -{ - "nodes": [ - { - "opr": { - "opr": { - "Scan": { - "scan_opt": 0, - "alias": { - "item": { - "Id": 0 - } - }, - "params": { - "tables": [ - { - "item": { - "Id": 0 - } - }, - { - "item": { - "Id": 1 - } - } - ], - "columns": [], - "is_all_columns": false, - "limit": null, - "predicate": null, - "sample_ratio": 1.0, - "extra": {} - }, - "idx_predicate": null, - "is_count_only": false, - "meta_data": { - "type": { - "type": { - "GraphType": { - "element_opt": 0, - "graph_data_type": [ - { - "label": { - "label": 1, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "lang" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "creationDate" - } - }, - "type": 12 - } - ] - }, - { - "label": { - "label": 0, - "src_label": null, - "dst_label": null - }, - "props": [ - { - "prop_id": { - "item": { - "Name": "id" - } - }, - "type": 2 - }, - { - "prop_id": { - "item": { - "Name": "name" - } - }, - "type": 4 - }, - { - "prop_id": { - "item": { - "Name": "age" - } - }, - "type": 1 - } - ] - } - ] - } - } - }, - "alias": 0 - } - } - } - }, - "children": [ - 1 - ] - }, - { - "opr": { - "opr": { - "GroupBy": { - "mappings": [], - "functions": [ - { - "vars": [ - { - "tag": { - "item": { - "Id": 0 - } - }, - "property": { - "item": { - "Key": { - "item": { - "Name": "name" - } - } - } - }, - "node_type": { - "type": { - "DataType": 4 - } - } - }, - { - "tag": { - "item": { - "Id": 0 - } - }, - "property": { - "item": { - "Key": { - "item": { - "Name": "age" - } - } - } - }, - "node_type": { - "type": { - "DataType": 1 - } - } - } - ], - "aggregate": 4, - "alias": { - "item": { - "Id": 1 - } - } - } - ], - "meta_data": [ - { - "type": { - "type": { - "DataType": 2 - } - }, - "alias": 1 - } - ] - } - } - }, - "children": [ - 2 - ] - }, - { - "opr": { - "opr": { - "Sink": { - "tags": [], - "sink_target": { - "inner": { - "SinkDefault": { - "id_name_mappings": [] - } - } - } - } - } - }, - "children": [] - } - ], - "roots": [ - 0 - ] -} diff --git a/interactive_engine/compiler/src/test/resources/proto/aggregate_test.json b/interactive_engine/compiler/src/test/resources/proto/aggregate_test.json index 86723fb046a3..da05e6fc976b 100644 --- a/interactive_engine/compiler/src/test/resources/proto/aggregate_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/aggregate_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -51,7 +60,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "alias": 1 @@ -70,17 +84,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -93,12 +116,19 @@ }, "metaData": [{ "type": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } }, "alias": 1 }, { "type": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } }, "alias": 2 }] diff --git a/interactive_engine/compiler/src/test/resources/proto/aggregate_test_2.json b/interactive_engine/compiler/src/test/resources/proto/aggregate_test_2.json index 6f0366b65d1d..45d12648a5df 100644 --- a/interactive_engine/compiler/src/test/resources/proto/aggregate_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/aggregate_test_2.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -66,7 +75,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -87,7 +98,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "alias": 2 @@ -102,7 +118,9 @@ } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }, "alias": 3 @@ -118,7 +136,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, { "tag": { @@ -130,7 +153,9 @@ } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }], "aggregate": "TO_LIST", @@ -140,17 +165,34 @@ }, "metaData": [{ "type": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } }, "alias": 2 }, { "type": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } }, "alias": 3 }, { "type": { - "dataType": "STRING_ARRAY" + "dataType": { + "array": { + "componentType": { + "string": { + "longText": { + } + } + }, + "maxLength": 2147483647 + } + } }, "alias": 4 }] diff --git a/interactive_engine/compiler/src/test/resources/proto/date_minus_date.json b/interactive_engine/compiler/src/test/resources/proto/date_minus_date.json index 422abd3637fc..50dae3b30854 100644 --- a/interactive_engine/compiler/src/test/resources/proto/date_minus_date.json +++ b/interactive_engine/compiler/src/test/resources/proto/date_minus_date.json @@ -12,15 +12,27 @@ } }, "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }, "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }, { "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } }, "dateTimeMinus": { "interval": "MILLISECOND" @@ -36,25 +48,39 @@ } }, "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }, "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }, { "brace": "RIGHT_BRACE" }, { "arith": "DIV", "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, { "const": { "i32": 1000 }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }] -} +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/proto/date_plus_literal_interval.json b/interactive_engine/compiler/src/test/resources/proto/date_plus_literal_interval.json index 06a79bd40432..ca347897f00b 100644 --- a/interactive_engine/compiler/src/test/resources/proto/date_plus_literal_interval.json +++ b/interactive_engine/compiler/src/test/resources/proto/date_plus_literal_interval.json @@ -7,22 +7,39 @@ } }, "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }, "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }, { "arith": "ADD", "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }, { "brace": "LEFT_BRACE" }, { "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } }, "timeInterval": { "const": { @@ -32,11 +49,15 @@ }, { "arith": "ADD", "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, { "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } }, "timeInterval": { "interval": "MONTH", @@ -47,4 +68,4 @@ }, { "brace": "RIGHT_BRACE" }] -} +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/proto/date_plus_param_interval.json b/interactive_engine/compiler/src/test/resources/proto/date_plus_param_interval.json index d3d674e5af7a..4cd8d71fcc5e 100644 --- a/interactive_engine/compiler/src/test/resources/proto/date_plus_param_interval.json +++ b/interactive_engine/compiler/src/test/resources/proto/date_plus_param_interval.json @@ -7,16 +7,31 @@ } }, "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }, "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }, { "arith": "ADD", "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }, { "brace": "LEFT_BRACE" @@ -24,29 +39,39 @@ "param": { "name": "year", "dataType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, { "arith": "ADD", "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, { "param": { "name": "month", "index": 1, "dataType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, { "brace": "RIGHT_BRACE" }] -} +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/proto/dedup_test_1.json b/interactive_engine/compiler/src/test/resources/proto/dedup_test_1.json index d19ad8cccadb..58f224a64a10 100644 --- a/interactive_engine/compiler/src/test/resources/proto/dedup_test_1.json +++ b/interactive_engine/compiler/src/test/resources/proto/dedup_test_1.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -53,11 +62,21 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }] }, @@ -73,7 +92,12 @@ "id": 1 }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/dedup_test_2.json b/interactive_engine/compiler/src/test/resources/proto/dedup_test_2.json index 724da2c6c941..502ee7a9bdf9 100644 --- a/interactive_engine/compiler/src/test/resources/proto/dedup_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/dedup_test_2.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -50,7 +59,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }] } @@ -65,17 +79,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/edge_expand_degree_test.json b/interactive_engine/compiler/src/test/resources/proto/edge_expand_degree_test.json index 3d26da0d02ca..7459f00b2790 100644 --- a/interactive_engine/compiler/src/test/resources/proto/edge_expand_degree_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/edge_expand_degree_test.json @@ -20,17 +20,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -62,7 +71,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -75,7 +86,9 @@ "functions": [{ "vars": [{ "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }], "alias": 0 @@ -84,7 +97,9 @@ }, "metaData": [{ "type": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }] }, { diff --git a/interactive_engine/compiler/src/test/resources/proto/edge_expand_test.json b/interactive_engine/compiler/src/test/resources/proto/edge_expand_test.json index 012ed5797438..e279bb865379 100644 --- a/interactive_engine/compiler/src/test/resources/proto/edge_expand_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/edge_expand_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -65,7 +74,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_filter_test.json b/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_filter_test.json index 9ecbf3cfe5ec..b0c055103c9a 100644 --- a/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_filter_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_filter_test.json @@ -20,17 +20,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -61,7 +70,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -85,23 +96,31 @@ } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "logical": "EQ", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "const": { "i32": 10 }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }] }, @@ -120,17 +139,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_test.json b/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_test.json index bbe42e4abf29..699a09fbe5e9 100644 --- a/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_test.json @@ -20,17 +20,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -62,7 +71,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_with_filters_test.json b/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_with_filters_test.json index 2df0b7aa68ea..40f8cd0bd470 100644 --- a/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_with_filters_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/edge_expand_vertex_with_filters_test.json @@ -20,17 +20,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -53,23 +62,31 @@ } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }, { "logical": "EQ", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "const": { "f64": 0.5 }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }] }, @@ -90,7 +107,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -114,23 +133,31 @@ } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "logical": "EQ", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "const": { "i32": 10 }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }] }, @@ -149,17 +176,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/expression_with_brace.json b/interactive_engine/compiler/src/test/resources/proto/expression_with_brace.json index 56ceee01be3a..b0612e4f2341 100644 --- a/interactive_engine/compiler/src/test/resources/proto/expression_with_brace.json +++ b/interactive_engine/compiler/src/test/resources/proto/expression_with_brace.json @@ -10,16 +10,22 @@ } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "arith": "SUB", "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "brace": "LEFT_BRACE" @@ -28,12 +34,16 @@ "i32": 1 }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "arith": "SUB", "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "var": { @@ -46,11 +56,15 @@ } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "brace": "RIGHT_BRACE" diff --git a/interactive_engine/compiler/src/test/resources/proto/extract_year.json b/interactive_engine/compiler/src/test/resources/proto/extract_year.json index 6b4efe09790d..0590f50afca7 100644 --- a/interactive_engine/compiler/src/test/resources/proto/extract_year.json +++ b/interactive_engine/compiler/src/test/resources/proto/extract_year.json @@ -4,7 +4,9 @@ "interval": "DAY" }, "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, { "var": { @@ -14,11 +16,21 @@ } }, "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }, "nodeType": { - "dataType": "DATE32" + "dataType": { + "temporal": { + "date32": { + } + } + } } }] -} +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/proto/filter_test.json b/interactive_engine/compiler/src/test/resources/proto/filter_test.json index 959f76895e29..b899f1e5bd74 100644 --- a/interactive_engine/compiler/src/test/resources/proto/filter_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/filter_test.json @@ -16,23 +16,40 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, { "logical": "EQ", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "const": { "str": "marko" }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }] }, @@ -50,17 +67,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/filter_test_2.json b/interactive_engine/compiler/src/test/resources/proto/filter_test_2.json index e3864551d52a..861f8054ecd5 100644 --- a/interactive_engine/compiler/src/test/resources/proto/filter_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/filter_test_2.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -66,7 +75,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -88,16 +99,22 @@ } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "logical": "GT", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "var": { @@ -110,11 +127,15 @@ } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }] } @@ -133,7 +154,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/getv_test.json b/interactive_engine/compiler/src/test/resources/proto/getv_test.json index 019ba459844a..ac44ac88c6ec 100644 --- a/interactive_engine/compiler/src/test/resources/proto/getv_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/getv_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -65,7 +74,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -94,17 +105,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/getv_with_filter_test.json b/interactive_engine/compiler/src/test/resources/proto/getv_with_filter_test.json index d0d3240ca53f..3535e0c8540d 100644 --- a/interactive_engine/compiler/src/test/resources/proto/getv_with_filter_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/getv_with_filter_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -65,7 +74,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -94,17 +105,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -125,23 +145,31 @@ } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "logical": "EQ", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "const": { "i32": 10 }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }] }, @@ -159,17 +187,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/id_in_list_expr.json b/interactive_engine/compiler/src/test/resources/proto/id_in_list_expr.json index 4b954a82154d..bbb0088858bd 100644 --- a/interactive_engine/compiler/src/test/resources/proto/id_in_list_expr.json +++ b/interactive_engine/compiler/src/test/resources/proto/id_in_list_expr.json @@ -6,16 +6,22 @@ } }, "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, { "logical": "WITHIN", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "const": { @@ -24,7 +30,9 @@ } }, "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/intersect_test.json b/interactive_engine/compiler/src/test/resources/proto/intersect_test.json index 75d2a411c153..765a746145a6 100644 --- a/interactive_engine/compiler/src/test/resources/proto/intersect_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/intersect_test.json @@ -22,42 +22,76 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -91,7 +125,9 @@ "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }] }, { "label": { @@ -102,7 +138,9 @@ "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }] }] } @@ -200,42 +238,76 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -248,7 +320,9 @@ }, "metaData": [{ "type": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } }, "alias": 3 }] @@ -262,4 +336,4 @@ } } }] -} +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/proto/intersect_test_2.json b/interactive_engine/compiler/src/test/resources/proto/intersect_test_2.json index 95399e202519..04e34dff15bd 100644 --- a/interactive_engine/compiler/src/test/resources/proto/intersect_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/intersect_test_2.json @@ -22,42 +22,76 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -95,7 +129,9 @@ "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }] }, { "label": { @@ -106,7 +142,9 @@ "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }] }] } @@ -138,42 +176,76 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "imageFile" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "language" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "content" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "length" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }, { "label": { @@ -183,32 +255,56 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "content" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "length" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -277,17 +373,29 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "url" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -349,17 +457,29 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "url" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -389,42 +509,76 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -437,7 +591,9 @@ }, "metaData": [{ "type": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } }, "alias": 6 }] @@ -451,4 +607,4 @@ } } }] -} +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/proto/join_test_1.json b/interactive_engine/compiler/src/test/resources/proto/join_test_1.json index 3c0be266b6b9..9a7305bc6ede 100644 --- a/interactive_engine/compiler/src/test/resources/proto/join_test_1.json +++ b/interactive_engine/compiler/src/test/resources/proto/join_test_1.json @@ -12,7 +12,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }], "rightKeys": [{ @@ -25,7 +30,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }], "leftPlan": { @@ -51,17 +61,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -92,17 +111,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/join_test_2.json b/interactive_engine/compiler/src/test/resources/proto/join_test_2.json index 76e9d5fe8679..856625a5ba09 100644 --- a/interactive_engine/compiler/src/test/resources/proto/join_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/join_test_2.json @@ -12,7 +12,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }], "rightKeys": [{ @@ -25,7 +30,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }], "leftPlan": { @@ -51,17 +61,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -96,7 +115,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -127,17 +148,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -170,22 +200,39 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lang" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }] }] } @@ -223,7 +270,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -253,17 +302,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/limit_test.json b/interactive_engine/compiler/src/test/resources/proto/limit_test.json index cbc7135f3b8b..bfb0fbbe8ea2 100644 --- a/interactive_engine/compiler/src/test/resources/proto/limit_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/limit_test.json @@ -20,17 +20,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/name_not_containing_expr.json b/interactive_engine/compiler/src/test/resources/proto/name_not_containing_expr.json index 7b5ea627e997..d5ff0a449601 100644 --- a/interactive_engine/compiler/src/test/resources/proto/name_not_containing_expr.json +++ b/interactive_engine/compiler/src/test/resources/proto/name_not_containing_expr.json @@ -2,7 +2,9 @@ "operators": [{ "logical": "NOT", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "brace": "LEFT_BRACE" @@ -14,23 +16,40 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, { "logical": "REGEX", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "const": { "str": ".*mar.*" }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, { "brace": "RIGHT_BRACE" diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_aggregate_test.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_aggregate_test.json index 6efc88dd5d12..9762bc924543 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_aggregate_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_aggregate_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -73,7 +82,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "alias": 1 @@ -92,17 +106,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -115,12 +138,19 @@ }, "metaData": [{ "type": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } }, "alias": 1 }, { "type": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } }, "alias": 2 }] diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_aggregate_test_2.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_aggregate_test_2.json deleted file mode 100644 index 3b786b2e1b05..000000000000 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_aggregate_test_2.json +++ /dev/null @@ -1,195 +0,0 @@ -{ - "plan": [{ - "opr": { - "scan": { - "alias": 0, - "params": { - "tables": [{ - "id": 0 - }], - "sampleRatio": 1.0 - } - } - }, - "metaData": [{ - "type": { - "graphType": { - "graphDataType": [{ - "label": { - }, - "props": [{ - "propId": { - "name": "id" - }, - "type": "INT64" - }, { - "propId": { - "name": "name" - }, - "type": "STRING" - }, { - "propId": { - "name": "age" - }, - "type": "INT32" - }] - }] - } - } - }] - }, { - "opr": { - "repartition": { - "toAnother": { - } - } - } - }, { - "opr": { - "edge": { - "params": { - "tables": [{ - "id": 0 - }], - "columns": [{ - "name": "weight" - }], - "sampleRatio": 1.0 - }, - "alias": 1, - "expandOpt": "EDGE" - } - }, - "metaData": [{ - "type": { - "graphType": { - "elementOpt": "EDGE", - "graphDataType": [{ - "label": { - "srcLabel": 0, - "dstLabel": 0 - }, - "props": [{ - "propId": { - "name": "weight" - }, - "type": "DOUBLE" - }] - }] - } - }, - "alias": 1 - }] - }, { - "opr": { - "repartition": { - "toAnother": { - "shuffleKey": 0 - } - } - } - }, { - "opr": { - "vertex": { - "tag": 0, - "opt": "ITSELF", - "params": { - "columns": [{ - "name": "name" - }], - "sampleRatio": 1.0 - }, - "alias": 0 - } - } - }, { - "opr": { - "groupBy": { - "mappings": [{ - "key": { - "tag": { - "id": 0 - }, - "property": { - "key": { - "name": "name" - } - }, - "nodeType": { - "dataType": "STRING" - } - }, - "alias": 2 - }, { - "key": { - "tag": { - "id": 1 - }, - "property": { - "key": { - "name": "weight" - } - }, - "nodeType": { - "dataType": "DOUBLE" - } - }, - "alias": 3 - }], - "functions": [{ - "vars": [{ - "tag": { - "id": 0 - }, - "property": { - "key": { - "name": "name" - } - }, - "nodeType": { - "dataType": "STRING" - } - }, { - "tag": { - "id": 1 - }, - "property": { - "key": { - "name": "weight" - } - }, - "nodeType": { - "dataType": "DOUBLE" - } - }], - "aggregate": "TO_LIST", - "alias": 4 - }] - } - }, - "metaData": [{ - "type": { - "dataType": "STRING" - }, - "alias": 2 - }, { - "type": { - "dataType": "DOUBLE" - }, - "alias": 3 - }, { - "type": { - }, - "alias": 4 - }] - }, { - "opr": { - "sink": { - "sinkTarget": { - "sinkDefault": { - } - } - } - } - }] -} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_dedup_test_1.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_dedup_test_1.json index 044b7d64c330..14f6a66b6650 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_dedup_test_1.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_dedup_test_1.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -75,11 +84,21 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }] }, @@ -95,7 +114,12 @@ "id": 1 }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_dedup_test_2.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_dedup_test_2.json index 78f170cc1291..0a8e55ce5007 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_dedup_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_dedup_test_2.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -72,7 +81,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }] } @@ -87,17 +101,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_degree_test.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_degree_test.json index dd438fb4be1a..f6d56c191649 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_degree_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_degree_test.json @@ -20,17 +20,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -69,7 +78,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -82,7 +93,9 @@ "functions": [{ "vars": [{ "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }], "alias": 0 @@ -91,7 +104,9 @@ }, "metaData": [{ "type": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }] }, { diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_test.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_test.json index 864e887cd7ef..3a873117084d 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -72,7 +81,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_vertex_filter_test.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_vertex_filter_test.json index 421c740fc602..d0890b1e434f 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_vertex_filter_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_vertex_filter_test.json @@ -20,17 +20,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -68,7 +77,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -99,23 +110,31 @@ } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "logical": "EQ", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "const": { "i32": 10 }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }] }, @@ -134,17 +153,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_vertex_test.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_vertex_test.json index 054044601b5b..a5f23f138ebf 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_vertex_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_edge_expand_vertex_test.json @@ -20,17 +20,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -69,7 +78,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_filter_test_2.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_filter_test_2.json index fb01b4e7ed8b..8f990e4d060a 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_filter_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_filter_test_2.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -73,7 +82,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -103,16 +114,22 @@ } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "logical": "GT", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "var": { @@ -125,11 +142,15 @@ } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }] } @@ -148,7 +169,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_getv_test.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_getv_test.json index e1856e2114ac..41904b58c7f4 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_getv_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_getv_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -72,7 +81,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -101,17 +112,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test.json index 002f573adf14..b7b1d0023904 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test.json @@ -22,42 +22,76 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -99,7 +133,9 @@ "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }] }, { "label": { @@ -110,7 +146,9 @@ "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }] }] } @@ -224,42 +262,76 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -272,7 +344,9 @@ }, "metaData": [{ "type": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } }, "alias": 3 }] @@ -286,4 +360,4 @@ } } }] -} +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test_2.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test_2.json index eacf38f51d36..75a0af64b6b0 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test_2.json @@ -22,42 +22,76 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -103,7 +137,9 @@ "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }] }, { "label": { @@ -114,7 +150,9 @@ "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }] }] } @@ -146,42 +184,76 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "imageFile" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "language" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "content" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "length" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }, { "label": { @@ -191,32 +263,56 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "content" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "length" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -293,17 +389,29 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "url" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -373,17 +481,29 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "url" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -413,42 +533,76 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -461,7 +615,9 @@ }, "metaData": [{ "type": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } }, "alias": 6 }] @@ -475,4 +631,4 @@ } } }] -} +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_join_test_1.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_join_test_1.json index 5e13bd2762fe..90aa72a75413 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_join_test_1.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_join_test_1.json @@ -12,7 +12,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }], "rightKeys": [{ @@ -25,7 +30,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }], "leftPlan": { @@ -51,17 +61,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -114,17 +133,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_join_test_2.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_join_test_2.json index 9bb6e40ca3bf..e7d15aa0ee21 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_join_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_join_test_2.json @@ -12,7 +12,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }], "rightKeys": [{ @@ -25,7 +30,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }], "leftPlan": { @@ -51,17 +61,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -103,7 +122,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -134,17 +155,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -199,22 +229,39 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lang" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "creationDate" }, - "type": "DATE32" + "type": { + "temporal": { + "date32": { + } + } + } }] }] } @@ -259,7 +306,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -289,17 +338,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_project_test.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_project_test.json index 2a2af6d9b8e0..b644efa893a6 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_project_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_project_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -61,11 +70,21 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }] }, @@ -75,7 +94,12 @@ }, "metaData": [{ "type": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } }, "alias": 1 }] diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_project_test_2.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_project_test_2.json index f8840b20e38b..1566c2bbeb6f 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_project_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_project_test_2.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -73,7 +82,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -104,11 +115,21 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }] }, @@ -126,11 +147,15 @@ } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }] }, @@ -140,12 +165,19 @@ }, "metaData": [{ "type": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } }, "alias": 2 }, { "type": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } }, "alias": 3 }] diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_sort_test.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_sort_test.json index 1a4eb6e52390..def04317eabf 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_sort_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_sort_test.json @@ -20,17 +20,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -67,7 +76,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "order": "ASC" diff --git a/interactive_engine/compiler/src/test/resources/proto/path_expand_test.json b/interactive_engine/compiler/src/test/resources/proto/path_expand_test.json index 304c237dfe7c..12220429df83 100644 --- a/interactive_engine/compiler/src/test/resources/proto/path_expand_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/path_expand_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -84,7 +93,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/path_fused_expand_test.json b/interactive_engine/compiler/src/test/resources/proto/path_fused_expand_test.json index 2f5ed2d97ff4..2130612d4151 100644 --- a/interactive_engine/compiler/src/test/resources/proto/path_fused_expand_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/path_fused_expand_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -71,7 +80,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/project_test.json b/interactive_engine/compiler/src/test/resources/proto/project_test.json index 8449baf778ba..a303fbd56a28 100644 --- a/interactive_engine/compiler/src/test/resources/proto/project_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/project_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -53,11 +62,21 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }] }, @@ -67,7 +86,12 @@ }, "metaData": [{ "type": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } }, "alias": 1 }] diff --git a/interactive_engine/compiler/src/test/resources/proto/project_test_2.json b/interactive_engine/compiler/src/test/resources/proto/project_test_2.json index 63d2a05c8233..d3f8843f1a1d 100644 --- a/interactive_engine/compiler/src/test/resources/proto/project_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/project_test_2.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -66,7 +75,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } @@ -89,11 +100,21 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }] }, @@ -111,11 +132,15 @@ } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }, "nodeType": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } } }] }, @@ -125,12 +150,19 @@ }, "metaData": [{ "type": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } }, "alias": 2 }, { "type": { - "dataType": "DOUBLE" + "dataType": { + "primitiveType": "DT_DOUBLE" + } }, "alias": 3 }] diff --git a/interactive_engine/compiler/src/test/resources/proto/scan_edge_test.json b/interactive_engine/compiler/src/test/resources/proto/scan_edge_test.json index 0211c4994efc..0aa63c7bef5f 100644 --- a/interactive_engine/compiler/src/test/resources/proto/scan_edge_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/scan_edge_test.json @@ -28,7 +28,9 @@ "propId": { "name": "weight" }, - "type": "DOUBLE" + "type": { + "primitiveType": "DT_DOUBLE" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/scan_filter_test.json b/interactive_engine/compiler/src/test/resources/proto/scan_filter_test.json index 9bb3096c2fac..7ee4f7e4a479 100644 --- a/interactive_engine/compiler/src/test/resources/proto/scan_filter_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/scan_filter_test.json @@ -16,23 +16,31 @@ } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }, { "logical": "EQ", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "const": { "i32": 10 }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }] }, @@ -50,17 +58,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/scan_test.json b/interactive_engine/compiler/src/test/resources/proto/scan_test.json index 11fe411986ac..d19dc9a0161e 100644 --- a/interactive_engine/compiler/src/test/resources/proto/scan_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/scan_test.json @@ -21,17 +21,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } diff --git a/interactive_engine/compiler/src/test/resources/proto/sort_test.json b/interactive_engine/compiler/src/test/resources/proto/sort_test.json index d8741401371f..b846542a639b 100644 --- a/interactive_engine/compiler/src/test/resources/proto/sort_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/sort_test.json @@ -20,17 +20,26 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "name" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "age" }, - "type": "INT32" + "type": { + "primitiveType": "DT_SIGNED_INT32" + } }] }] } @@ -48,7 +57,12 @@ } }, "nodeType": { - "dataType": "STRING" + "dataType": { + "string": { + "longText": { + } + } + } } }, "order": "ASC" diff --git a/interactive_engine/compiler/src/test/resources/proto/st_path_test.json b/interactive_engine/compiler/src/test/resources/proto/st_path_test.json index 480246a1ea93..3dc5ca23c98d 100644 --- a/interactive_engine/compiler/src/test/resources/proto/st_path_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/st_path_test.json @@ -16,42 +16,73 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -71,42 +102,73 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -150,42 +212,73 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -212,23 +305,31 @@ } }, "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, { "logical": "EQ", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "const": { "i32": 2012 }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }] }, @@ -269,7 +370,9 @@ "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }] }] } @@ -300,42 +403,73 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -382,42 +516,73 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -446,23 +611,31 @@ } }, "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, "nodeType": { - "dataType": "INT64" + "dataType": { + "primitiveType": "DT_SIGNED_INT64" + } } }, { "logical": "EQ", "nodeType": { - "dataType": "BOOLEAN" + "dataType": { + "primitiveType": "DT_BOOL" + } } }, { "const": { "i32": 2012 }, "nodeType": { - "dataType": "INT32" + "dataType": { + "primitiveType": "DT_SIGNED_INT32" + } } }] }, @@ -502,7 +675,9 @@ "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }] }] } @@ -533,42 +708,73 @@ "propId": { "name": "id" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "firstName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "lastName" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "gender" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "birthday" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "creationDate" }, - "type": "INT64" + "type": { + "primitiveType": "DT_SIGNED_INT64" + } }, { "propId": { "name": "locationIP" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }, { "propId": { "name": "browserUsed" }, - "type": "STRING" + "type": { + "string": { + "longText": { + } + } + } }] }] } @@ -658,4 +864,4 @@ } } }] -} +} \ No newline at end of file From fc6820b978c3512a6a4508fef8d1ca67cefeb999 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Mon, 30 Dec 2024 16:43:48 +0800 Subject: [PATCH 09/23] support flex data type in ir physical plan --- .../meta/procedure/StoredProcedureMeta.java | 31 +- .../ir/meta/reader/HttpIrMetaReader.java | 1 + .../ir/meta/reader/LocalIrMetaReader.java | 1 + .../ir/meta/schema/GSDataTypeConvertor.java | 258 --------- .../common/ir/meta/schema/GraphOptTable.java | 32 +- .../ir/meta/schema/IrDataTypeConvertor.java | 511 ++++++++++++++++++ .../ir/meta/schema/IrGraphProperty.java | 36 ++ .../common/ir/meta/schema/IrGraphSchema.java | 6 +- .../common/ir/meta/schema/SchemaSpec.java | 9 +- .../ir/meta/schema/SchemaSpecManager.java | 6 +- .../common/ir/meta/schema/Utils.java | 93 ++-- .../common/ir/runtime/proto/Utils.java | 6 +- .../ir/tools/GraphStdOperatorTable.java | 4 +- .../sql/type/GraphOperandMetaDataImpl.java | 19 +- .../calcite/sql/type/GraphOperandTypes.java | 18 +- .../common/config/YamlConfigTest.java | 16 +- .../graphscope/common/ir/ExpressionTest.java | 2 +- .../graphscope/common/ir/ProjectTest.java | 2 +- .../common/ir/TypeConvertorTest.java | 56 ++ .../ir/planner/rbo/FieldTrimmerTest.java | 295 ---------- .../cypher/antlr4/CallProcedureTest.java | 2 +- .../graphscope/cypher/antlr4/MatchTest.java | 6 +- .../graphscope/cypher/antlr4/WhereTest.java | 2 +- .../graphscope/cypher/antlr4/WithTest.java | 2 +- .../gremlin/antlr4x/GraphBuilderTest.java | 6 +- .../test/resources/config/modern/graph.yaml | 3 + .../src/test/resources/proto/filter_test.json | 3 +- .../proto/name_not_containing_expr.json | 5 +- .../src/test/resources/schema/type_test.yaml | 37 ++ 29 files changed, 774 insertions(+), 694 deletions(-) delete mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphProperty.java create mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/TypeConvertorTest.java delete mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/FieldTrimmerTest.java create mode 100644 interactive_engine/compiler/src/test/resources/schema/type_test.yaml diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java index 7a4377014e72..c835bb613de7 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java @@ -20,8 +20,8 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.IrMetaStats; -import com.alibaba.graphscope.common.ir.meta.schema.GSDataTypeConvertor; import com.alibaba.graphscope.common.ir.meta.schema.GSDataTypeDesc; +import com.alibaba.graphscope.common.ir.meta.schema.IrDataTypeConvertor; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphStatistics; import com.alibaba.graphscope.common.ir.meta.schema.SchemaSpec; import com.alibaba.graphscope.common.ir.rex.RexProcedureCall; @@ -128,10 +128,17 @@ public Mode getMode() { public static class Parameter { private final String name; private final RelDataType dataType; + // allow cast among types in the same family, i.e. int32 to int64, char to varchar + private final boolean allowCast; public Parameter(String name, RelDataType dataType) { + this(name, dataType, false); + } + + public Parameter(String name, RelDataType dataType, boolean allowCast) { this.name = name; this.dataType = dataType; + this.allowCast = allowCast; } public String getName() { @@ -142,6 +149,10 @@ public RelDataType getDataType() { return dataType; } + public boolean allowCast() { + return allowCast; + } + @Override public String toString() { return "Parameter{" + "name='" + name + '\'' + ", dataType=" + dataType + '}'; @@ -171,8 +182,8 @@ public static void perform(StoredProcedureMeta meta, OutputStream outputStream) } private static Map createProduceMetaMap(StoredProcedureMeta meta) { - GSDataTypeConvertor typeConvertor = - new GSDataTypeConvertor.Calcite(typeFactory); + IrDataTypeConvertor typeConvertor = + new IrDataTypeConvertor.Flex(typeFactory); return ImmutableMap.of( Config.NAME.getKey(), meta.name, @@ -216,8 +227,8 @@ private static Map createProduceMetaMap(StoredProcedureMeta meta public static class Deserializer { public static StoredProcedureMeta perform(InputStream inputStream) throws IOException { - GSDataTypeConvertor typeConvertor = - new GSDataTypeConvertor.Calcite(typeFactory); + IrDataTypeConvertor typeConvertor = + new IrDataTypeConvertor.Flex(typeFactory); Yaml yaml = new Yaml(); Map config = yaml.load(inputStream); return new StoredProcedureMeta( @@ -240,7 +251,7 @@ private static T getValue( } private static RelDataType createReturnType( - List config, GSDataTypeConvertor typeConvertor) { + List config, IrDataTypeConvertor typeConvertor) { List fields = Lists.newArrayList(); if (config == null) { return new RelRecordType(fields); @@ -262,7 +273,7 @@ private static RelDataType createReturnType( } private static List createParameters( - List config, GSDataTypeConvertor typeConvertor) { + List config, IrDataTypeConvertor typeConvertor) { List parameters = Lists.newArrayList(); if (config == null) { return parameters; @@ -270,12 +281,16 @@ private static List createParameters( Iterator iterator = config.iterator(); while (iterator.hasNext()) { Map field = (Map) iterator.next(); + Object castValue = field.get("allow_cast"); + boolean allowCast = + castValue == null ? false : Boolean.valueOf(String.valueOf(castValue)); parameters.add( new StoredProcedureMeta.Parameter( (String) field.get("name"), typeConvertor.convert( new GSDataTypeDesc( - (Map) field.get("type"))))); + (Map) field.get("type"))), + allowCast)); } return parameters; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java index ba6cfb089d87..d683f1f00636 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java @@ -72,6 +72,7 @@ public IrMeta readMeta() throws IOException { metaPair.getValue0(), SnapshotId.createEmpty(), // todo: return snapshot id from http service new IrGraphSchema( + configs, new SchemaInputStream( new ByteArrayInputStream( metaInYaml.getBytes(StandardCharsets.UTF_8)), diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java index 3b859755e4f8..a6a7e0c4e83e 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java @@ -60,6 +60,7 @@ public IrMeta readMeta() throws IOException { : SchemaSpec.Type.IR_CORE_IN_JSON; IrGraphSchema graphSchema = new IrGraphSchema( + configs, new SchemaInputStream( new FileInputStream(schemaPath.toFile()), schemaSpec)); IrMeta irMeta = diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java deleted file mode 100644 index 444add5f15cc..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeConvertor.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * - * * Copyright 2020 Alibaba Group Holding Limited. - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package com.alibaba.graphscope.common.ir.meta.schema; - -import com.alibaba.graphscope.common.ir.type.ArbitraryArrayType; -import com.alibaba.graphscope.common.ir.type.ArbitraryMapType; -import com.alibaba.graphscope.common.ir.type.GraphLabelType; -import com.alibaba.graphscope.groot.common.schema.wrapper.DataType; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; - -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.sql.type.IntervalSqlType; -import org.apache.calcite.sql.type.SqlTypeName; - -import java.util.Map; - -/** - * define the convertor from flex type system to the given target type system - * @param - */ -public interface GSDataTypeConvertor { - // convert from flex type system to target type system - T convert(GSDataTypeDesc from); - - // convert from target type system to flex type system - GSDataTypeDesc convert(T from); - - class Groot implements GSDataTypeConvertor { - @Override - public DataType convert(GSDataTypeDesc from) { - Map typeMap = from.getYamlDesc(); - Object value; - if ((value = typeMap.get("primitive_type")) != null) { - switch (value.toString()) { - case "DT_SIGNED_INT32": - return DataType.INT; - case "DT_SIGNED_INT64": - return DataType.LONG; - case "DT_BOOL": - return DataType.BOOL; - case "DT_FLOAT": - return DataType.FLOAT; - case "DT_DOUBLE": - return DataType.DOUBLE; - default: - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to DataType"); - } - } else if ((value = typeMap.get("string")) != null) { - Map strType = (Map) value; - if (strType.containsKey("long_text")) { - return DataType.STRING; - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to DataType"); - } - } else if ((value = typeMap.get("temporal")) != null) { - Map temporalType = (Map) value; - if (temporalType.containsKey("date32")) { - return DataType.DATE; - } else if (temporalType.containsKey("time32")) { - return DataType.TIME32; - } else if (temporalType.containsKey("timestamp")) { - return DataType.TIMESTAMP; - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to DataType"); - } - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to DataType"); - } - } - - @Override - public GSDataTypeDesc convert(DataType from) { - throw new UnsupportedOperationException( - "convert from DataType to GSDataTypeDesc is unsupported yet"); - } - } - - class Calcite implements GSDataTypeConvertor { - private final RelDataTypeFactory typeFactory; - - public Calcite(RelDataTypeFactory typeFactory) { - this.typeFactory = typeFactory; - } - - @Override - public RelDataType convert(GSDataTypeDesc from) { - Map typeMap = from.getYamlDesc(); - Object value; - if ((value = typeMap.get("primitive_type")) != null) { - switch (value.toString()) { - case "DT_ANY": - return typeFactory.createSqlType(SqlTypeName.ANY); - case "DT_SIGNED_INT32": - return typeFactory.createSqlType(SqlTypeName.INTEGER); - case "DT_SIGNED_INT64": - return typeFactory.createSqlType(SqlTypeName.BIGINT); - case "DT_BOOL": - return typeFactory.createSqlType(SqlTypeName.BOOLEAN); - case "DT_FLOAT": - return typeFactory.createSqlType(SqlTypeName.FLOAT); - case "DT_DOUBLE": - return typeFactory.createSqlType(SqlTypeName.DOUBLE); - default: - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to RelDataType"); - } - } else if ((value = typeMap.get("string")) != null) { - Map strType = (Map) value; - if (strType.containsKey("long_text")) { - return typeFactory.createSqlType(SqlTypeName.CHAR); - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to RelDataType"); - } - } else if ((value = typeMap.get("temporal")) != null) { - Map temporalType = (Map) value; - if (temporalType.containsKey("date32")) { - return typeFactory.createSqlType(SqlTypeName.DATE); - } else if (temporalType.containsKey("time32")) { - return typeFactory.createSqlType(SqlTypeName.TIME); - } else if (temporalType.containsKey("timestamp")) { - return typeFactory.createSqlType(SqlTypeName.TIMESTAMP); - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to RelDataType"); - } - } else if ((value = typeMap.get("array")) != null) { - Map arrayType = (Map) value; - Map componentType = - (Map) arrayType.get("component_type"); - Preconditions.checkArgument( - componentType != null, "field 'component_type' is required in array type"); - return typeFactory.createArrayType(convert(new GSDataTypeDesc(componentType)), -1); - } else if ((value = typeMap.get("map")) != null) { - Map mapType = (Map) value; - Map keyType = (Map) mapType.get("key_type"); - Preconditions.checkArgument( - keyType != null, "field 'key_type' is required in map type"); - Map valueType = (Map) mapType.get("value_type"); - Preconditions.checkArgument( - valueType != null, "field 'value_type' is required in map type"); - return typeFactory.createMapType( - convert(new GSDataTypeDesc(keyType)), - convert(new GSDataTypeDesc(valueType))); - } else { - throw new UnsupportedOperationException( - "can not convert GSDataTypeDesc [" + from + "] to RelDataType"); - } - } - - @Override - public GSDataTypeDesc convert(RelDataType from) { - if (from instanceof IntervalSqlType) { - return new GSDataTypeDesc(ImmutableMap.of("primitive_type", "DT_SIGNED_INT64")); - } else if (from instanceof GraphLabelType) { - return new GSDataTypeDesc(ImmutableMap.of("primitive_type", "DT_SIGNED_INT32")); - } - SqlTypeName typeName = from.getSqlTypeName(); - Map yamlDesc; - switch (typeName) { - case INTEGER: - yamlDesc = ImmutableMap.of("primitive_type", "DT_SIGNED_INT32"); - break; - case BIGINT: - yamlDesc = ImmutableMap.of("primitive_type", "DT_SIGNED_INT64"); - break; - case BOOLEAN: - yamlDesc = ImmutableMap.of("primitive_type", "DT_BOOL"); - break; - case FLOAT: - yamlDesc = ImmutableMap.of("primitive_type", "DT_FLOAT"); - break; - case DOUBLE: - yamlDesc = ImmutableMap.of("primitive_type", "DT_DOUBLE"); - break; - case CHAR: - Map longTextMap = Maps.newHashMap(); - longTextMap.put("long_text", ImmutableMap.of()); - yamlDesc = ImmutableMap.of("string", longTextMap); - break; - case DATE: - Map dateMap = Maps.newHashMap(); - dateMap.put("date32", ImmutableMap.of()); - yamlDesc = ImmutableMap.of("temporal", dateMap); - break; - case TIME: - Map timeMap = Maps.newHashMap(); - timeMap.put("time32", ImmutableMap.of()); - yamlDesc = ImmutableMap.of("temporal", timeMap); - break; - case TIMESTAMP: - Map timestampMap = Maps.newHashMap(); - timestampMap.put("timestamp", ImmutableMap.of()); - yamlDesc = ImmutableMap.of("temporal", timestampMap); - break; - case ARRAY: - case MULTISET: - Map componentType; - if (from instanceof ArbitraryArrayType) { - componentType = ImmutableMap.of("primitive_type", "DT_ANY"); - } else { - componentType = convert(from.getComponentType()).getYamlDesc(); - } - yamlDesc = - ImmutableMap.of( - "array", - ImmutableMap.of( - "component_type", - componentType, - "max_length", - Integer.MAX_VALUE)); - break; - case MAP: - Map keyType; - Map valueType; - if (from instanceof ArbitraryMapType) { - keyType = ImmutableMap.of("primitive_type", "DT_ANY"); - valueType = ImmutableMap.of("primitive_type", "DT_ANY"); - } else { - keyType = convert(from.getKeyType()).getYamlDesc(); - valueType = convert(from.getValueType()).getYamlDesc(); - } - yamlDesc = - ImmutableMap.of( - "map", - ImmutableMap.of("key_type", keyType, "value_type", valueType)); - break; - default: - throw new UnsupportedOperationException( - "can not convert RelDataType [" + from + "] to GSDataTypeDesc"); - } - return new GSDataTypeDesc(yamlDesc); - } - } -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptTable.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptTable.java index 1757d19ba08a..b48368c2b2d5 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptTable.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptTable.java @@ -22,6 +22,7 @@ import com.alibaba.graphscope.common.ir.type.GraphLabelType; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; import com.alibaba.graphscope.groot.common.schema.api.*; +import com.alibaba.graphscope.groot.common.schema.wrapper.DataType; import com.google.common.collect.Lists; import org.apache.calcite.linq4j.tree.Expression; @@ -37,7 +38,6 @@ import org.apache.calcite.rel.type.RelDataTypeFieldImpl; import org.apache.calcite.schema.ColumnStrategy; import org.apache.calcite.schema.Statistic; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ImmutableBitSet; import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; @@ -135,33 +135,13 @@ private List getUniqueKeys( } private RelDataType deriveType(GraphProperty property) { + if (property instanceof IrGraphProperty) { + return ((IrGraphProperty) property).getRelDataType(); + } RelDataTypeFactory typeFactory = this.schema.getTypeFactory(); requireNonNull(typeFactory, "typeFactory"); - switch (property.getDataType()) { - case BOOL: - return typeFactory.createSqlType(SqlTypeName.BOOLEAN); - case CHAR: - case STRING: - return typeFactory.createSqlType(SqlTypeName.CHAR); - case SHORT: - case INT: - return typeFactory.createSqlType(SqlTypeName.INTEGER); - case LONG: - return typeFactory.createSqlType(SqlTypeName.BIGINT); - case FLOAT: - return typeFactory.createSqlType(SqlTypeName.FLOAT); - case DOUBLE: - return typeFactory.createSqlType(SqlTypeName.DOUBLE); - case DATE: - return typeFactory.createSqlType(SqlTypeName.DATE); - case TIME32: - return typeFactory.createSqlType(SqlTypeName.TIME); - case TIMESTAMP: - return typeFactory.createSqlType(SqlTypeName.TIMESTAMP); - default: - throw new UnsupportedOperationException( - "type " + property.getDataType().name() + " not supported"); - } + IrDataTypeConvertor typeConvertor = new IrDataTypeConvertor.Groot(typeFactory); + return typeConvertor.convert(property.getDataType()); } @Override diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java new file mode 100644 index 000000000000..b6d079786e33 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java @@ -0,0 +1,511 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta.schema; + +import static java.util.Objects.requireNonNull; + +import com.alibaba.graphscope.common.ir.type.ArbitraryArrayType; +import com.alibaba.graphscope.common.ir.type.ArbitraryMapType; +import com.alibaba.graphscope.common.ir.type.GraphLabelType; +import com.alibaba.graphscope.groot.common.schema.wrapper.DataType; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.type.IntervalSqlType; +import org.apache.calcite.sql.type.SqlTypeName; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Objects; + +/** + * Define the mutual type conversion between other type systems and Ir type system {@code RelDataType} + */ +public interface IrDataTypeConvertor { + Logger logger = LoggerFactory.getLogger(IrDataTypeConvertor.class); + + RelDataType convert(T dataFrom); + + T convert(RelDataType dataFrom); + + class Groot implements IrDataTypeConvertor { + private final RelDataTypeFactory typeFactory; + + public Groot(RelDataTypeFactory typeFactory) { + this.typeFactory = typeFactory; + } + + @Override + public RelDataType convert(DataType from) { + requireNonNull(typeFactory, "typeFactory should not be null"); + switch (from) { + case BOOL: + return typeFactory.createSqlType(SqlTypeName.BOOLEAN); + case CHAR: + // single character + return typeFactory.createSqlType(SqlTypeName.CHAR, 1); + case STRING: + // string with unlimited length + return typeFactory.createSqlType( + SqlTypeName.VARCHAR, RelDataType.PRECISION_NOT_SPECIFIED); + case SHORT: + // 2-bytes signed integer + return typeFactory.createSqlType(SqlTypeName.SMALLINT); + case INT: + // 4-bytes signed integer + return typeFactory.createSqlType(SqlTypeName.INTEGER); + case LONG: + // 8-bytes signed integer + return typeFactory.createSqlType(SqlTypeName.BIGINT); + case FLOAT: + // single precision floating point, 4 bytes + return typeFactory.createSqlType(SqlTypeName.FLOAT); + case DOUBLE: + // double precision floating point, 8 bytes + return typeFactory.createSqlType(SqlTypeName.DOUBLE); + case DATE: + // int32 days since 1970-01-01 + return typeFactory.createSqlType(SqlTypeName.DATE); + case TIME32: + // int32 milliseconds past midnight + return typeFactory.createSqlType(SqlTypeName.TIME); + case TIMESTAMP: + // int64 milliseconds since 1970-01-01 00:00:00.000000 + return typeFactory.createSqlType(SqlTypeName.TIMESTAMP); + case INT_LIST: + // array of 4-bytes signed integer, unlimited size + return typeFactory.createArrayType( + convert(DataType.INT), RelDataType.PRECISION_NOT_SPECIFIED); + case LONG_LIST: + // array of 8-bytes signed integer, unlimited size + return typeFactory.createArrayType( + convert(DataType.LONG), RelDataType.PRECISION_NOT_SPECIFIED); + case FLOAT_LIST: + // array of single precision floating point, unlimited size + return typeFactory.createArrayType( + convert(DataType.FLOAT), RelDataType.PRECISION_NOT_SPECIFIED); + case DOUBLE_LIST: + // array of double precision floating point, unlimited size + return typeFactory.createArrayType( + convert(DataType.DOUBLE), RelDataType.PRECISION_NOT_SPECIFIED); + case STRING_LIST: + // array of string, unlimited size + return typeFactory.createArrayType( + convert(DataType.STRING), RelDataType.PRECISION_NOT_SPECIFIED); + case UNKNOWN: + return typeFactory.createSqlType(SqlTypeName.UNKNOWN); + case BYTES: + case BYTES_LIST: + default: + throw new UnsupportedOperationException( + "convert GrootDataType [" + + from.name() + + "] to RelDataType is unsupported yet"); + } + } + + @Override + public DataType convert(RelDataType dataFrom) { + SqlTypeName typeName = dataFrom.getSqlTypeName(); + switch (typeName) { + case BOOLEAN: + return DataType.BOOL; + case CHAR: + if (dataFrom.getPrecision() == 1) { + return DataType.CHAR; + } + case VARCHAR: + if (dataFrom.getPrecision() == RelDataType.PRECISION_NOT_SPECIFIED) { + return DataType.STRING; + } + case SMALLINT: + return DataType.SHORT; + case INTEGER: + return DataType.INT; + case BIGINT: + return DataType.LONG; + case FLOAT: + return DataType.FLOAT; + case DOUBLE: + return DataType.DOUBLE; + case DATE: + return DataType.DATE; + case TIME: + return DataType.TIME32; + case TIMESTAMP: + return DataType.TIMESTAMP; + case MULTISET: + case ARRAY: + RelDataType componentType = dataFrom.getComponentType(); + // check the array or set is a unlimited size list of primitive type + if (componentType != null + && dataFrom.getPrecision() == RelDataType.PRECISION_NOT_SPECIFIED) { + switch (componentType.getSqlTypeName()) { + case INTEGER: + return DataType.INT_LIST; + case BIGINT: + return DataType.LONG_LIST; + case FLOAT: + return DataType.FLOAT_LIST; + case DOUBLE: + return DataType.DOUBLE_LIST; + case VARCHAR: + if (componentType.getPrecision() + == RelDataType.PRECISION_NOT_SPECIFIED) { + return DataType.STRING_LIST; + } + } + } + case UNKNOWN: + default: + return DataType.UNKNOWN; + } + } + } + + class Flex implements IrDataTypeConvertor { + private final RelDataTypeFactory typeFactory; + + public Flex(RelDataTypeFactory typeFactory) { + this.typeFactory = typeFactory; + } + + @Override + public RelDataType convert(GSDataTypeDesc from) { + Objects.requireNonNull(typeFactory, "typeFactory should not be null"); + Map typeMap = from.getYamlDesc(); + Object value; + if ((value = typeMap.get("primitive_type")) != null) { + switch (value.toString()) { + case "DT_ANY": + // any type + return typeFactory.createSqlType(SqlTypeName.ANY); + case "DT_SIGNED_INT32": + // 4-bytes signed integer + return typeFactory.createSqlType(SqlTypeName.INTEGER); + case "DT_SIGNED_INT64": + // 8-bytes signed integer + return typeFactory.createSqlType(SqlTypeName.BIGINT); + case "DT_BOOL": + // boolean type + return typeFactory.createSqlType(SqlTypeName.BOOLEAN); + case "DT_FLOAT": + // single precision floating point, 4 bytes + return typeFactory.createSqlType(SqlTypeName.FLOAT); + case "DT_DOUBLE": + // double precision floating point, 8 bytes + return typeFactory.createSqlType(SqlTypeName.DOUBLE); + } + } else if ((value = typeMap.get("string")) != null) { + Map strType = (Map) value; + if (strType.containsKey("long_text")) { + // string with unlimited length + return typeFactory.createSqlType( + SqlTypeName.VARCHAR, RelDataType.PRECISION_NOT_SPECIFIED); + } else if (strType.containsKey("char")) { + Object charValue = strType.get("char"); + Integer fixedLen = getIntValue(charValue, "fixed_length"); + if (fixedLen == null) { + fixedLen = + typeFactory.getTypeSystem().getDefaultPrecision(SqlTypeName.CHAR); + logger.warn( + "can not convert {} to a valid fixed length, use default" + + " length {} instead", + charValue, + fixedLen); + } + // string with fixed length + return typeFactory.createSqlType(SqlTypeName.CHAR, fixedLen); + } else if (strType.containsKey("var_char")) { + Object varCharValue = strType.get("var_char"); + Integer maxLen = getIntValue(varCharValue, "max_length"); + if (maxLen == null) { + maxLen = + typeFactory + .getTypeSystem() + .getDefaultPrecision(SqlTypeName.VARCHAR); + logger.warn( + "can not convert {} to a valid max length, use default" + + " length {} instead", + varCharValue, + maxLen); + } + // string with variable length, bound by max length + return typeFactory.createSqlType(SqlTypeName.VARCHAR, maxLen); + } + } else if ((value = typeMap.get("temporal")) != null) { + Map temporalType = (Map) value; + if (temporalType.containsKey("date32")) { + // int32 days since 1970-01-01 + return typeFactory.createSqlType(SqlTypeName.DATE); + } else if (temporalType.containsKey("time32")) { + // int32 milliseconds past midnight + return typeFactory.createSqlType(SqlTypeName.TIME); + } else if (temporalType.containsKey("timestamp")) { + // int64 milliseconds since 1970-01-01 00:00:00.000000 + return typeFactory.createSqlType(SqlTypeName.TIMESTAMP); + } + } else if ((value = typeMap.get("array")) != null) { + Map arrayType = (Map) value; + Map componentType = + (Map) arrayType.get("component_type"); + Preconditions.checkArgument( + componentType != null, "field 'component_type' is required in array type"); + // array of component type, unlimited size + return typeFactory.createArrayType( + convert(new GSDataTypeDesc(componentType)), + RelDataType.PRECISION_NOT_SPECIFIED); + } else if ((value = typeMap.get("map")) != null) { + Map mapType = (Map) value; + Map keyType = (Map) mapType.get("key_type"); + Preconditions.checkArgument( + keyType != null, "field 'key_type' is required in map type"); + Map valueType = (Map) mapType.get("value_type"); + Preconditions.checkArgument( + valueType != null, "field 'value_type' is required in map type"); + // map of key type to value type + return typeFactory.createMapType( + convert(new GSDataTypeDesc(keyType)), + convert(new GSDataTypeDesc(valueType))); + } else if ((value = typeMap.get("decimal")) != null) { + Integer precision = getIntValue(value, "precision"); + if (precision == null) { + precision = + typeFactory.getTypeSystem().getDefaultPrecision(SqlTypeName.DECIMAL); + logger.warn( + "can not convert {} to a valid precision, use default" + + " precision {} instead", + value, + precision); + } + Integer scale = getIntValue(value, "scale"); + if (scale == null) { + scale = typeFactory.getTypeSystem().getMaxScale(SqlTypeName.DECIMAL); + logger.warn( + "can not convert {} to a valid scale, use max" + " scale {} instead", + value, + scale); + } + // decimal type with precision and scale + return typeFactory.createSqlType(SqlTypeName.DECIMAL, precision, scale); + } + throw new UnsupportedOperationException( + "convert GSDataTypeDesc [" + from + "] to RelDataType is unsupported yet"); + } + + @Override + public GSDataTypeDesc convert(RelDataType from) { + if (from instanceof IntervalSqlType) { + return new GSDataTypeDesc(ImmutableMap.of("primitive_type", "DT_SIGNED_INT64")); + } else if (from instanceof GraphLabelType) { + return new GSDataTypeDesc(ImmutableMap.of("primitive_type", "DT_SIGNED_INT32")); + } + SqlTypeName typeName = from.getSqlTypeName(); + Map yamlDesc; + switch (typeName) { + case INTEGER: + yamlDesc = ImmutableMap.of("primitive_type", "DT_SIGNED_INT32"); + break; + case BIGINT: + yamlDesc = ImmutableMap.of("primitive_type", "DT_SIGNED_INT64"); + break; + case BOOLEAN: + yamlDesc = ImmutableMap.of("primitive_type", "DT_BOOL"); + break; + case FLOAT: + yamlDesc = ImmutableMap.of("primitive_type", "DT_FLOAT"); + break; + case DOUBLE: + yamlDesc = ImmutableMap.of("primitive_type", "DT_DOUBLE"); + break; + case CHAR: + Map charMap = Maps.newHashMap(); + charMap.put("char", ImmutableMap.of("fixed_length", from.getPrecision())); + yamlDesc = ImmutableMap.of("string", charMap); + break; + case VARCHAR: + if (from.getPrecision() == RelDataType.PRECISION_NOT_SPECIFIED) { + Map longTextMap = Maps.newHashMap(); + longTextMap.put("long_text", ImmutableMap.of()); + yamlDesc = ImmutableMap.of("string", longTextMap); + } else { + Map varCharMap = Maps.newHashMap(); + varCharMap.put( + "var_char", ImmutableMap.of("max_length", from.getPrecision())); + yamlDesc = ImmutableMap.of("string", varCharMap); + } + break; + case DATE: + Map dateMap = Maps.newHashMap(); + dateMap.put("date32", ImmutableMap.of()); + yamlDesc = ImmutableMap.of("temporal", dateMap); + break; + case TIME: + Map timeMap = Maps.newHashMap(); + timeMap.put("time32", ImmutableMap.of()); + yamlDesc = ImmutableMap.of("temporal", timeMap); + break; + case TIMESTAMP: + Map timestampMap = Maps.newHashMap(); + timestampMap.put("timestamp", ImmutableMap.of()); + yamlDesc = ImmutableMap.of("temporal", timestampMap); + break; + case ARRAY: + case MULTISET: + Map componentType; + if (from instanceof ArbitraryArrayType) { + componentType = ImmutableMap.of("primitive_type", "DT_ANY"); + } else { + componentType = convert(from.getComponentType()).getYamlDesc(); + } + yamlDesc = + ImmutableMap.of( + "array", + ImmutableMap.of( + "component_type", + componentType, + "max_length", + Integer.MAX_VALUE)); + break; + case MAP: + Map keyType; + Map valueType; + if (from instanceof ArbitraryMapType) { + keyType = ImmutableMap.of("primitive_type", "DT_ANY"); + valueType = ImmutableMap.of("primitive_type", "DT_ANY"); + } else { + keyType = convert(from.getKeyType()).getYamlDesc(); + valueType = convert(from.getValueType()).getYamlDesc(); + } + yamlDesc = + ImmutableMap.of( + "map", + ImmutableMap.of("key_type", keyType, "value_type", valueType)); + break; + case DECIMAL: + yamlDesc = + ImmutableMap.of( + "decimal", + ImmutableMap.of( + "precision", from.getPrecision(), + "scale", from.getScale())); + break; + default: + throw new UnsupportedOperationException( + "convert RelDataType [" + + from + + "] to GSDataTypeDesc is unsupported yet"); + } + return new GSDataTypeDesc(yamlDesc); + } + + private @Nullable Integer getIntValue(Object valueMap, String key) { + if (valueMap instanceof Map) { + Object value = ((Map) valueMap).get(key); + if (value != null) { + if (value instanceof Number) { + return ((Number) value).intValue(); + } else if (value instanceof String) { + return Integer.parseInt((String) value); + } + } + } + return null; + } + } + + class IrCoreOrdinal implements IrDataTypeConvertor { + private final RelDataTypeFactory typeFactory; + + public IrCoreOrdinal(RelDataTypeFactory typeFactory) { + this.typeFactory = typeFactory; + } + + @Override + public RelDataType convert(Integer ordinal) { + switch (ordinal) { + case 0: + // boolean type + return typeFactory.createSqlType(SqlTypeName.BOOLEAN); + case 1: + // 4-bytes signed integer + return typeFactory.createSqlType((SqlTypeName.INTEGER)); + case 2: + // 8-bytes signed integer + return typeFactory.createSqlType(SqlTypeName.BIGINT); + case 3: + // double precision floating point, 8 bytes + return typeFactory.createSqlType(SqlTypeName.DOUBLE); + case 4: + // string with unlimited length + return typeFactory.createSqlType( + SqlTypeName.VARCHAR, RelDataType.PRECISION_NOT_SPECIFIED); + case 5: + // binary data with unlimited length + return typeFactory.createSqlType(SqlTypeName.BINARY); + case 6: + // array of 4-bytes signed integer, unlimited size + return typeFactory.createArrayType( + typeFactory.createSqlType(SqlTypeName.INTEGER), + RelDataType.PRECISION_NOT_SPECIFIED); + case 7: + // array of 8-bytes signed integer, unlimited size + return typeFactory.createArrayType( + typeFactory.createSqlType(SqlTypeName.BIGINT), + RelDataType.PRECISION_NOT_SPECIFIED); + case 8: + // array of double precision floating point, unlimited size + return typeFactory.createArrayType( + typeFactory.createSqlType(SqlTypeName.DOUBLE), + RelDataType.PRECISION_NOT_SPECIFIED); + case 9: + // array of string, unlimited size + return typeFactory.createArrayType( + typeFactory.createSqlType( + SqlTypeName.VARCHAR, RelDataType.PRECISION_NOT_SPECIFIED), + RelDataType.PRECISION_NOT_SPECIFIED); + case 12: + // int32 days since 1970-01-01 + return typeFactory.createSqlType(SqlTypeName.DATE); + case 13: + // int32 milliseconds past midnight + return typeFactory.createSqlType(SqlTypeName.TIME); + case 14: + // int64 milliseconds since 1970-01-01 00:00:00.000000 + return typeFactory.createSqlType(SqlTypeName.TIMESTAMP); + default: + throw new UnsupportedOperationException( + "convert IrCoreDataType [" + + ordinal + + "] to RelDataType is unsupported yet"); + } + } + + @Override + public Integer convert(RelDataType from) { + throw new UnsupportedOperationException( + "convert RelDataType to IrCoreDataType is unsupported yet"); + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphProperty.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphProperty.java new file mode 100644 index 000000000000..d610bbe57051 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphProperty.java @@ -0,0 +1,36 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta.schema; + +import com.alibaba.graphscope.groot.common.schema.impl.DefaultGraphProperty; + +import org.apache.calcite.rel.type.RelDataType; + +public class IrGraphProperty extends DefaultGraphProperty { + private final RelDataType relDataType; + + public IrGraphProperty(int id, String name, RelDataType relDataType) { + super(id, name, new IrDataTypeConvertor.Groot(null).convert(relDataType)); + this.relDataType = relDataType; + } + + public RelDataType getRelDataType() { + return this.relDataType; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java index b8f5ce184096..11b34449bc20 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java @@ -16,6 +16,8 @@ package com.alibaba.graphscope.common.ir.meta.schema; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.type.GraphTypeFactoryImpl; import com.alibaba.graphscope.groot.common.exception.PropertyNotFoundException; import com.alibaba.graphscope.groot.common.exception.TypeNotFoundException; import com.alibaba.graphscope.groot.common.schema.api.*; @@ -33,14 +35,14 @@ public class IrGraphSchema implements GraphSchema { private final boolean isColumnId; private final SchemaSpecManager specManager; - public IrGraphSchema(SchemaInputStream schemaInputStream) throws IOException { + public IrGraphSchema(Configs configs, SchemaInputStream schemaInputStream) throws IOException { this.isColumnId = false; String content = new String( schemaInputStream.getInputStream().readAllBytes(), StandardCharsets.UTF_8); schemaInputStream.getInputStream().close(); SchemaSpec spec = new SchemaSpec(schemaInputStream.getType(), content); - this.graphSchema = spec.convert(); + this.graphSchema = spec.convert(new GraphTypeFactoryImpl(configs)); this.specManager = new SchemaSpecManager(this, spec); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/SchemaSpec.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/SchemaSpec.java index dc35c7069760..f7b3c06862e7 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/SchemaSpec.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/SchemaSpec.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.yaml.snakeyaml.Yaml; import java.util.Map; @@ -36,19 +37,19 @@ public SchemaSpec(Type type, String content) { this.content = content; } - public GraphSchema convert() throws JacksonException { + public GraphSchema convert(RelDataTypeFactory typeFactory) throws JacksonException { switch (type) { case IR_CORE_IN_JSON: - return Utils.buildSchemaFromJson(content); + return Utils.buildSchemaFromJson(content, typeFactory); case FLEX_IN_YAML: - return Utils.buildSchemaFromYaml(content); + return Utils.buildSchemaFromYaml(content, typeFactory); case FLEX_IN_JSON: default: ObjectMapper mapper = new ObjectMapper(); JsonNode rootNode = mapper.readTree(content); Map rootMap = mapper.convertValue(rootNode, Map.class); Yaml yaml = new Yaml(); - return Utils.buildSchemaFromYaml(yaml.dump(rootMap)); + return Utils.buildSchemaFromYaml(yaml.dump(rootMap), typeFactory); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/SchemaSpecManager.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/SchemaSpecManager.java index 80079cbff9eb..8803a30d4e1c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/SchemaSpecManager.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/SchemaSpecManager.java @@ -81,7 +81,7 @@ public SchemaSpec getSpec(SchemaSpec.Type type) { IrSchemaParser.getInstance() .parse(parent.getGraphSchema(), parent.isColumnId())); case FLEX_IN_JSON: - if (source.getType() == SchemaSpec.Type.FLEX_IN_YAML) { + if (source != null && source.getType() == SchemaSpec.Type.FLEX_IN_YAML) { Yaml yaml = new Yaml(); Map rootMap = yaml.load(source.getContent()); ObjectMapper mapper = new ObjectMapper(); @@ -91,7 +91,7 @@ public SchemaSpec getSpec(SchemaSpec.Type type) { return null; case FLEX_IN_YAML: default: - if (source.getType() == SchemaSpec.Type.FLEX_IN_JSON) { + if (source != null && source.getType() == SchemaSpec.Type.FLEX_IN_JSON) { ObjectMapper mapper = new ObjectMapper(); JsonNode rootNode = mapper.readTree(source.getContent()); Map rootMap = mapper.convertValue(rootNode, Map.class); @@ -104,7 +104,7 @@ public SchemaSpec getSpec(SchemaSpec.Type type) { } catch (Exception e) { logger.warn( "can not convert from {} to {} due to some unexpected exception:", - source.getType(), + source == null ? null : source.getType(), target, e); return null; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java index 0d1bffe79161..0e8d1a10da72 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/Utils.java @@ -18,7 +18,6 @@ import com.alibaba.graphscope.groot.common.schema.api.*; import com.alibaba.graphscope.groot.common.schema.impl.*; -import com.alibaba.graphscope.groot.common.schema.wrapper.DataType; import com.alibaba.graphscope.groot.common.schema.wrapper.EdgeKind; import com.alibaba.graphscope.groot.common.schema.wrapper.LabelId; import com.fasterxml.jackson.databind.JsonNode; @@ -27,6 +26,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.yaml.snakeyaml.Yaml; import java.util.Iterator; @@ -41,7 +41,8 @@ public abstract class Utils { * @param schemaYaml * @return */ - public static final GraphSchema buildSchemaFromYaml(String schemaYaml) { + public static final GraphSchema buildSchemaFromYaml( + String schemaYaml, RelDataTypeFactory typeFactory) { Yaml yaml = new Yaml(); Map yamlAsMap = yaml.load(schemaYaml); Map schemaMap = @@ -51,8 +52,9 @@ public static final GraphSchema buildSchemaFromYaml(String schemaYaml) { Map vertexMap = Maps.newHashMap(); Map edgeMap = Maps.newHashMap(); Map propNameToIdMap = Maps.newHashMap(); - GSDataTypeConvertor typeConvertor = new GSDataTypeConvertor.Groot(); - builderGraphElementFromYaml( + IrDataTypeConvertor typeConvertor = + new IrDataTypeConvertor.Flex(typeFactory); + buildGraphElementFromYaml( (List) Objects.requireNonNull( schemaMap.get("vertex_types"), @@ -63,7 +65,7 @@ public static final GraphSchema buildSchemaFromYaml(String schemaYaml) { propNameToIdMap, typeConvertor); if (schemaMap.get("edge_types") != null) { - builderGraphElementFromYaml( + buildGraphElementFromYaml( (List) Objects.requireNonNull( schemaMap.get("edge_types"), @@ -77,13 +79,13 @@ public static final GraphSchema buildSchemaFromYaml(String schemaYaml) { return new DefaultGraphSchema(vertexMap, edgeMap, propNameToIdMap); } - public static final void builderGraphElementFromYaml( + public static final void buildGraphElementFromYaml( List elementList, String type, Map vertexMap, Map edgeMap, Map propNameToIdMap, - GSDataTypeConvertor typeConvertor) { + IrDataTypeConvertor typeConvertor) { int curVertexTypeId = 0; int curEdgeTypeId = 0; for (Object element : elementList) { @@ -145,13 +147,20 @@ public static final void builderGraphElementFromYaml( } curPropertyId = propertyId++; propNameToIdMap.put(propertyName, curPropertyId); + Object propertyType = propertyMap.get("property_type"); + if (!(propertyType instanceof Map)) { + throw new IllegalArgumentException( + "invalid property type [" + + propertyType + + "] in flex yaml config"); + } propertyList.add( - new DefaultGraphProperty( + new IrGraphProperty( curPropertyId, propertyName, - toDataType( - propertyMap.get("property_type"), - typeConvertor))); + typeConvertor.convert( + new GSDataTypeDesc( + (Map) propertyType)))); } } } @@ -187,26 +196,35 @@ public static final void builderGraphElementFromYaml( } } - public static DataType toDataType(Object type, GSDataTypeConvertor typeConvertor) { - return typeConvertor.convert(new GSDataTypeDesc((Map) type)); - } - /** * build {@link GraphSchema} from schema json (in ir core format) * @param schemaJson * @return */ - public static final GraphSchema buildSchemaFromJson(String schemaJson) { + public static final GraphSchema buildSchemaFromJson( + String schemaJson, RelDataTypeFactory typeFactory) { ObjectMapper mapper = new ObjectMapper(); try { JsonNode jsonNode = mapper.readTree(schemaJson); Map vertexMap = Maps.newHashMap(); Map edgeMap = Maps.newHashMap(); Map propNameToIdMap = Maps.newHashMap(); + IrDataTypeConvertor typeConvertor = + new IrDataTypeConvertor.IrCoreOrdinal(typeFactory); buildGraphElementFromJson( - jsonNode.get("entities"), "VERTEX", vertexMap, edgeMap, propNameToIdMap); + jsonNode.get("entities"), + "VERTEX", + vertexMap, + edgeMap, + propNameToIdMap, + typeConvertor); buildGraphElementFromJson( - jsonNode.get("relations"), "EDGE", vertexMap, edgeMap, propNameToIdMap); + jsonNode.get("relations"), + "EDGE", + vertexMap, + edgeMap, + propNameToIdMap, + typeConvertor); return new DefaultGraphSchema(vertexMap, edgeMap, propNameToIdMap); } catch (Exception e) { throw new RuntimeException(e); @@ -218,7 +236,8 @@ public static final void buildGraphElementFromJson( String type, Map vertexMap, Map edgeMap, - Map propNameToIdMap) { + Map propNameToIdMap, + IrDataTypeConvertor typeConvertor) { Objects.requireNonNull(elementList); Iterator var1 = elementList.iterator(); @@ -239,7 +258,7 @@ public static final void buildGraphElementFromJson( propNameToIdMap.put(propName, propId); int propTypeId = column.get("data_type").asInt(); GraphProperty property = - new DefaultGraphProperty(propId, propName, toDataType(propTypeId)); + new IrGraphProperty(propId, propName, typeConvertor.convert(propTypeId)); propertyList.add(property); boolean isPrimaryKey = column.get("is_primary_key").asBoolean(); if (isPrimaryKey) { @@ -267,40 +286,6 @@ public static final void buildGraphElementFromJson( } } - public static final DataType toDataType(int ordinal) { - switch (ordinal) { - case 0: - return DataType.BOOL; - case 1: - return DataType.INT; - case 2: - return DataType.LONG; - case 3: - return DataType.DOUBLE; - case 4: - return DataType.STRING; - case 5: - return DataType.BYTES; - case 6: - return DataType.INT_LIST; - case 7: - return DataType.LONG_LIST; - case 8: - return DataType.DOUBLE_LIST; - case 9: - return DataType.STRING_LIST; - case 12: - return DataType.DATE; - case 13: - return DataType.TIME32; - case 14: - return DataType.TIMESTAMP; - default: - throw new UnsupportedOperationException( - "convert from ir core type " + ordinal + " to DataType is unsupported yet"); - } - } - /** * build {@link GraphStatistics} from statistics json * @param statisticsJson diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java index 3dd0f7a83675..e95e00a5930a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java @@ -16,7 +16,8 @@ package com.alibaba.graphscope.common.ir.runtime.proto; -import com.alibaba.graphscope.common.ir.meta.schema.GSDataTypeConvertor; +import com.alibaba.graphscope.common.ir.meta.schema.GSDataTypeDesc; +import com.alibaba.graphscope.common.ir.meta.schema.IrDataTypeConvertor; import com.alibaba.graphscope.common.ir.rel.type.group.GraphAggCall; import com.alibaba.graphscope.common.ir.rex.RexVariableAliasCollector; import com.alibaba.graphscope.common.ir.tools.AliasInference; @@ -274,7 +275,7 @@ public static final OuterExpression.ExprOpr protoOperator(SqlOperator operator) public static final com.alibaba.graphscope.proto.type.Common.DataType protoBasicDataType( RelDataType basicType) { try { - GSDataTypeConvertor convertor = new GSDataTypeConvertor.Calcite(null); + IrDataTypeConvertor convertor = new IrDataTypeConvertor.Flex(null); return convertor.convert(basicType).getProtoDesc(); } catch (Exception e) { throw new RuntimeException(e); @@ -312,6 +313,7 @@ public static final DataType.IrDataType protoIrDataType( SqlTypeName.INTEGER, SqlTypeName.BIGINT, SqlTypeName.CHAR, + SqlTypeName.VARCHAR, SqlTypeName.DECIMAL, SqlTypeName.FLOAT, SqlTypeName.DOUBLE); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java index b9b632171908..307ccf1dbd2e 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphStdOperatorTable.java @@ -302,7 +302,7 @@ public static final SqlFunction USER_DEFINED_FUNCTION(FunctionMeta meta) { SqlKind.OTHER, ReturnTypes.ARG0, null, - GraphOperandTypes.operandMetadata( + new GraphOperandMetaDataImpl( ImmutableList.of( GraphTypeFamily.PATH, SqlTypeFamily.IGNORE, @@ -325,7 +325,7 @@ public static final SqlFunction USER_DEFINED_FUNCTION(FunctionMeta meta) { SqlKind.OTHER, ReturnTypes.ARG2.andThen(SqlTypeTransforms.TO_ARRAY), null, - GraphOperandTypes.operandMetadata( + new GraphOperandMetaDataImpl( ImmutableList.of( GraphTypeFamily.PATH, SqlTypeFamily.IGNORE, SqlTypeFamily.ANY), typeFactory -> ImmutableList.of(), diff --git a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java index 5a9a93dc663f..685e62937684 100644 --- a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java +++ b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java @@ -39,25 +39,38 @@ public class GraphOperandMetaDataImpl extends GraphFamilyOperandTypeChecker implements SqlOperandMetadata { private final Function> paramTypesFactory; private final IntFunction paramNameFn; + private final Predicate allowCast; - GraphOperandMetaDataImpl( + public GraphOperandMetaDataImpl( List expectedFamilies, Function<@Nullable RelDataTypeFactory, List> paramTypesFactory, IntFunction paramNameFn, Predicate optional) { + this(expectedFamilies, paramTypesFactory, paramNameFn, optional, i -> false); + } + + public GraphOperandMetaDataImpl( + List expectedFamilies, + Function<@Nullable RelDataTypeFactory, List> paramTypesFactory, + IntFunction paramNameFn, + Predicate optional, + Predicate allowCast) { super(expectedFamilies, optional); this.paramTypesFactory = Objects.requireNonNull(paramTypesFactory, "paramTypesFactory"); this.paramNameFn = paramNameFn; + this.allowCast = allowCast; } @Override protected Collection getAllowedTypeNames( RelDataTypeFactory typeFactory, SqlTypeFamily family, int iFormalOperand) { + boolean allowCast = this.allowCast.test(iFormalOperand); + if (allowCast) { + return family.getTypeNames(); + } List paramsAllowedTypes = paramTypes(typeFactory); if (paramsAllowedTypes.size() > iFormalOperand) { return ImmutableList.of(paramsAllowedTypes.get(iFormalOperand).getSqlTypeName()); - } else if (expectedFamilies.get(iFormalOperand) instanceof SqlTypeFamily) { - return ((SqlTypeFamily) expectedFamilies.get(iFormalOperand)).getTypeNames(); } throw new IllegalArgumentException( "cannot find allowed type for type index=" diff --git a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java index 5fa8a8ec6fc3..7d5c4533a955 100644 --- a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java +++ b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java @@ -19,14 +19,9 @@ import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; import com.google.common.collect.ImmutableList; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeFamily; import java.util.List; -import java.util.function.Function; -import java.util.function.IntFunction; -import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -101,23 +96,16 @@ public static FamilyOperandTypeChecker family(RelDataTypeFamily... families) { public static final SqlSingleOperandTypeChecker DIVISION_OPERATOR = OperandTypes.or(NUMERIC_NUMERIC, INTERVAL_NUMERIC); - public static SqlOperandMetadata operandMetadata( - List families, - Function> typesFactory, - IntFunction operandName, - Predicate optional) { - return new GraphOperandMetaDataImpl(families, typesFactory, operandName, optional); - } - public static SqlOperandTypeChecker metaTypeChecker(StoredProcedureMeta meta) { List parameters = meta.getParameters(); - return GraphOperandTypes.operandMetadata( + return new GraphOperandMetaDataImpl( parameters.stream() .map(p -> p.getDataType().getSqlTypeName().getFamily()) .collect(Collectors.toList()), typeFactory -> parameters.stream().map(p -> p.getDataType()).collect(Collectors.toList()), i -> parameters.get(i).getName(), - i -> false); + i -> false, + i -> parameters.get(i).allowCast()); } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java index 4914e682ad21..7fc4f78a1f06 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java @@ -41,7 +41,7 @@ public void procedure_config_test() throws Exception { GraphStoredProcedures procedures = irMeta.getStoredProcedures(); StoredProcedureMeta meta = procedures.getStoredProcedure("ldbc_ic2"); Assert.assertEquals( - "StoredProcedureMeta{name='ldbc_ic2', returnType=RecordType(CHAR(1) name)," + "StoredProcedureMeta{name='ldbc_ic2', returnType=RecordType(VARCHAR name)," + " parameters=[Parameter{name='personId2', dataType=BIGINT}," + " Parameter{name='maxDate', dataType=BIGINT}], option={type=x_cypher," + " query=MATCH(n: PERSON ${personId2}) WHERE n.creationDate < ${maxDate}" @@ -80,17 +80,17 @@ public void schema_config_test() throws Exception { IrGraphSchema graphSchema = irMeta.getSchema(); Assert.assertEquals( "DefaultGraphVertex{labelId=0, label=person," - + " propertyList=[DefaultGraphProperty{id=0, name=id, dataType=LONG}," - + " DefaultGraphProperty{id=1, name=name, dataType=STRING}," - + " DefaultGraphProperty{id=2, name=age, dataType=INT}]," + + " propertyList=[IrGraphProperty{id=0, name=id, dataType=LONG}," + + " IrGraphProperty{id=1, name=name, dataType=STRING}," + + " IrGraphProperty{id=2, name=age, dataType=INT}]," + " primaryKeyList=[id]}", graphSchema.getElement("person").toString()); Assert.assertEquals( "DefaultGraphVertex{labelId=1, label=software," - + " propertyList=[DefaultGraphProperty{id=0, name=id, dataType=LONG}," - + " DefaultGraphProperty{id=1, name=name, dataType=STRING}," - + " DefaultGraphProperty{id=2, name=lang, dataType=STRING}," - + " DefaultGraphProperty{id=3, name=creationDate, dataType=DATE}]," + + " propertyList=[IrGraphProperty{id=0, name=id, dataType=LONG}," + + " IrGraphProperty{id=1, name=name, dataType=STRING}," + + " IrGraphProperty{id=2, name=lang, dataType=STRING}," + + " IrGraphProperty{id=3, name=creationDate, dataType=DATE}]," + " primaryKeyList=[id]}", graphSchema.getElement("software").toString()); } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java index dc254f246bdc..f5f74b51724c 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ExpressionTest.java @@ -86,7 +86,7 @@ public void variable_3_test() { @Test public void variable_4_test() { RexNode node = builder.source(mockSourceConfig(null)).variable(null, "name"); - Assert.assertEquals(node.getType().getSqlTypeName(), SqlTypeName.CHAR); + Assert.assertEquals(node.getType().getSqlTypeName(), SqlTypeName.VARCHAR); Assert.assertEquals("_.name", node.toString()); } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ProjectTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ProjectTest.java index 09793b294933..33dc642a5c2e 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ProjectTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/ProjectTest.java @@ -44,7 +44,7 @@ public void project_1_test() { Assert.assertEquals("[a]", project.getProjects().toString()); Assert.assertEquals( "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)], properties=[BIGINT id," - + " CHAR(1) name, INTEGER age]) a)", + + " VARCHAR name, INTEGER age]) a)", project.getRowType().toString()); } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/TypeConvertorTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/TypeConvertorTest.java new file mode 100644 index 000000000000..b2786edc6811 --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/TypeConvertorTest.java @@ -0,0 +1,56 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir; + +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; +import com.alibaba.graphscope.cypher.antlr4.Utils; + +import org.apache.calcite.rel.type.RelDataType; +import org.junit.Assert; +import org.junit.Test; + +public class TypeConvertorTest { + @Test + public void string_test() { + LogicalPlan plan = + Utils.evalLogicalPlan("Match (n) Return n.name1;", "schema/type_test.yaml"); + RelDataType type = plan.getOutputType(); + // convert long text to VARCHAR with unlimited length, the length is omitted + Assert.assertEquals("RecordType(VARCHAR name1)", type.toString()); + + plan = Utils.evalLogicalPlan("Match (n) Return n.name2", "schema/type_test.yaml"); + type = plan.getOutputType(); + // convert fixed length text to CHAR, the specified length is 255 + Assert.assertEquals("RecordType(CHAR(255) name2)", type.toString()); + + plan = Utils.evalLogicalPlan("Match (n) Return n.name3", "schema/type_test.yaml"); + type = plan.getOutputType(); + // convert varied length text to VARCHAR, the specified length is 255 + Assert.assertEquals("RecordType(VARCHAR(255) name3)", type.toString()); + } + + @Test + public void decimal_test() { + LogicalPlan plan = + Utils.evalLogicalPlan("Match (n) Return n.value1;", "schema/type_test.yaml"); + RelDataType type = plan.getOutputType(); + // convert decimal to DECIMAL, the specified precision is 4 and scale is 2 + Assert.assertEquals("RecordType(DECIMAL(4, 2) value1)", type.toString()); + } +} diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/FieldTrimmerTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/FieldTrimmerTest.java deleted file mode 100644 index c9e8bbabb709..000000000000 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/rbo/FieldTrimmerTest.java +++ /dev/null @@ -1,295 +0,0 @@ -package com.alibaba.graphscope.common.ir.planner.rbo; - -import com.alibaba.graphscope.common.ir.Utils; -import com.alibaba.graphscope.common.ir.planner.GraphFieldTrimmer; -import com.alibaba.graphscope.common.ir.rel.type.group.GraphAggCall; -import com.alibaba.graphscope.common.ir.tools.GraphBuilder; -import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; -import com.alibaba.graphscope.common.ir.tools.config.*; - -import org.apache.calcite.rel.RelNode; -import org.junit.Assert; -import org.junit.Test; - -import java.util.List; - -public class FieldTrimmerTest { - private String rowTypeString(RelNode rel) { - StringBuilder builder = new StringBuilder(); - rowTypeStringInternal(rel, builder); - return builder.toString(); - } - - private void rowTypeStringInternal(RelNode rel, StringBuilder builder) { - builder.append(rel.getRowType()).append("\n"); - for (RelNode node : rel.getInputs()) { - rowTypeStringInternal(node, builder); - } - } - - // Match(x:person)-[y:knows]->( ) where x.name = "Alice" return x.name as name , x.age as age - @Test - public void field_trimmer_1_test() { - GraphBuilder builder = Utils.mockGraphBuilder(); - RelNode sentence = - builder.source( - new SourceConfig( - GraphOpt.Source.VERTEX, - new LabelConfig(false).addLabel("person"), - "x")) - .expand( - new ExpandConfig( - GraphOpt.Expand.OUT, - new LabelConfig(false).addLabel("knows"), - "y")) - .build(); - RelNode project = - builder.match(sentence, GraphOpt.Match.INNER) - .filter( - builder.call( - GraphStdOperatorTable.EQUALS, - builder.variable("x", "name"), - builder.literal("Alice"))) - .project( - List.of( - builder.variable("x", "name"), - builder.variable("x", "age")), - List.of("name", "age")) - .build(); - - GraphFieldTrimmer trimmer = new GraphFieldTrimmer(builder); - RelNode after = trimmer.trim(project); - Assert.assertEquals( - "GraphLogicalProject(name=[x.name], age=[x.age], isAppend=[false])\n" - + " LogicalFilter(condition=[=(x.name, _UTF-8'Alice')])\n" - + " GraphLogicalSingleMatch(input=[null]," - + " sentence=[GraphLogicalExpand(tableConfig=[{isAll=false, tables=[knows]}]," - + " alias=[y], opt=[OUT])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," - + " alias=[x], opt=[VERTEX])\n" - + "], matchOpt=[INNER])", - after.explain().trim()); - - Assert.assertEquals( - "RecordType(CHAR(1) name, INTEGER age)\n" - + "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)]," - + " properties=[CHAR(1) name, INTEGER age]) x," - + " Graph_Schema_Type(labels=[EdgeLabel(knows, person, person)], properties=[])" - + " y)\n" - + "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)]," - + " properties=[CHAR(1) name, INTEGER age]) x," - + " Graph_Schema_Type(labels=[EdgeLabel(knows, person, person)], properties=[])" - + " y)\n", - rowTypeString(after)); - } - - // Match(x:person)-[y:knows]->(v:person) with x as p, x.age as age, v as friend where p.name = - // "Alice" and age > 10 - // return p - @Test - public void field_trimmer_2_test() { - GraphBuilder builder = Utils.mockGraphBuilder(); - RelNode sentence = - builder.source( - new SourceConfig( - GraphOpt.Source.VERTEX, - new LabelConfig(false).addLabel("person"), - "x")) - .expand( - new ExpandConfig( - GraphOpt.Expand.OUT, - new LabelConfig(false).addLabel("knows"), - "y")) - .getV( - new GetVConfig( - GraphOpt.GetV.END, - new LabelConfig(false).addLabel("person"), - "v")) - .build(); - RelNode project = - builder.match(sentence, GraphOpt.Match.INNER) - .project( - List.of( - builder.variable("x"), - builder.variable("x", "age"), - builder.variable("v")), - List.of("p", "age", "friend")) - .filter( - builder.call( - GraphStdOperatorTable.AND, - builder.call( - GraphStdOperatorTable.EQUALS, - builder.variable("p", "name"), - builder.literal("Alice")), - builder.call( - GraphStdOperatorTable.GREATER_THAN, - builder.variable("age"), - builder.literal("10")))) - .project(List.of(builder.variable("p"))) - .build(); - GraphFieldTrimmer trimmer = new GraphFieldTrimmer(builder); - RelNode after = trimmer.trim(project); - - // for column trimming - Assert.assertEquals( - "GraphLogicalProject(p=[p], isAppend=[false])\n" - + " LogicalFilter(condition=[AND(=(p.name, _UTF-8'Alice'), >(age," - + " _UTF-8'10'))])\n" - + " GraphLogicalProject(p=[x], age=[x.age], isAppend=[false])\n" - + " GraphLogicalSingleMatch(input=[null]," - + " sentence=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}]," - + " alias=[v], opt=[END])\n" - + " GraphLogicalExpand(tableConfig=[{isAll=false, tables=[knows]}], alias=[y]," - + " opt=[OUT])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," - + " alias=[x], opt=[VERTEX])\n" - + "], matchOpt=[INNER])", - after.explain().trim()); - Assert.assertEquals( - "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)], properties=[]) p)\n" - + "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)]," - + " properties=[CHAR(1) name]) p, INTEGER age)\n" - + "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)]," - + " properties=[CHAR(1) name]) p, INTEGER age)\n" - + "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)]," - + " properties=[CHAR(1) name, INTEGER age]) x," - + " Graph_Schema_Type(labels=[EdgeLabel(knows, person, person)], properties=[])" - + " y, Graph_Schema_Type(labels=[VertexLabel(person)], properties=[]) v)\n", - rowTypeString(after)); - // for property trimming - - } - - // Match(x:person)-[y:knows]->(friend:person) where x.age >10 return x, count(friend) as cnt - // order - // by cnt limit 3 - @Test - public void field_trimmer_3_test() { - GraphBuilder builder = Utils.mockGraphBuilder(); - RelNode sentence = - builder.source( - new SourceConfig( - GraphOpt.Source.VERTEX, - new LabelConfig(false).addLabel("person"), - "x")) - .expand( - new ExpandConfig( - GraphOpt.Expand.OUT, - new LabelConfig(false).addLabel("knows"), - "y")) - .getV( - new GetVConfig( - GraphOpt.GetV.END, - new LabelConfig(false).addLabel("person"), - "friend")) - .build(); - RelNode sort = - builder.match(sentence, GraphOpt.Match.INNER) - .filter( - builder.call( - GraphStdOperatorTable.GREATER_THAN, - builder.variable("x", "age"), - builder.literal(10))) - .aggregate( - builder.groupKey(builder.variable("x")), - List.of( - new GraphAggCall( - builder.getCluster(), - GraphStdOperatorTable.COUNT, - List.of(builder.variable("friend"))) - .as("cnt"))) - .sortLimit( - builder.literal(0), - builder.literal(3), - List.of(builder.variable("cnt"))) - .build(); - GraphFieldTrimmer trimmer = new GraphFieldTrimmer(builder); - RelNode after = trimmer.trim(sort); - - // for column trimming - Assert.assertEquals( - "GraphLogicalSort(sort0=[cnt], dir0=[ASC], offset=[0], fetch=[3])\n" - + " GraphLogicalAggregate(keys=[{variables=[x], aliases=[x]}]," - + " values=[[{operands=[friend], aggFunction=COUNT, alias='cnt'," - + " distinct=false}]])\n" - + " LogicalFilter(condition=[>(x.age, 10)])\n" - + " GraphLogicalSingleMatch(input=[null]," - + " sentence=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}]," - + " alias=[friend], opt=[END])\n" - + " GraphLogicalExpand(tableConfig=[{isAll=false, tables=[knows]}], alias=[y]," - + " opt=[OUT])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," - + " alias=[x], opt=[VERTEX])\n" - + "], matchOpt=[INNER])", - after.explain().trim()); - Assert.assertEquals( - "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)], properties=[]) x," - + " BIGINT cnt)\n" - + "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)], properties=[]) x," - + " BIGINT cnt)\n" - + "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)]," - + " properties=[INTEGER age]) x, Graph_Schema_Type(labels=[EdgeLabel(knows," - + " person, person)], properties=[]) y," - + " Graph_Schema_Type(labels=[VertexLabel(person)], properties=[]) friend)\n" - + "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)]," - + " properties=[INTEGER age]) x, Graph_Schema_Type(labels=[EdgeLabel(knows," - + " person, person)], properties=[]) y," - + " Graph_Schema_Type(labels=[VertexLabel(person)], properties=[]) friend)\n", - rowTypeString(after)); - } - - // Match(x:person)-[y:knows]->( ) with name=x.name where name = "Alice" return x.id - // test append=true - @Test - public void field_trimmer_4_test() { - GraphBuilder builder = Utils.mockGraphBuilder(); - RelNode sentence = - builder.source( - new SourceConfig( - GraphOpt.Source.VERTEX, - new LabelConfig(false).addLabel("person"), - "x")) - .expand( - new ExpandConfig( - GraphOpt.Expand.OUT, - new LabelConfig(false).addLabel("knows"), - "y")) - .build(); - RelNode project = - builder.match(sentence, GraphOpt.Match.INNER) - .project(List.of(builder.variable("x", "name")), List.of("name"), true) - .filter( - builder.call( - GraphStdOperatorTable.EQUALS, - builder.variable("name"), - builder.literal("Alice"))) - .project(List.of(builder.variable("x", "id")), List.of("id")) - .build(); - - GraphFieldTrimmer trimmer = new GraphFieldTrimmer(builder); - RelNode after = trimmer.trim(project); - Assert.assertEquals( - "GraphLogicalProject(id=[x.id], isAppend=[false])\n" - + " LogicalFilter(condition=[=(name, _UTF-8'Alice')])\n" - + " GraphLogicalProject(x=[x], name=[x.name], isAppend=[false])\n" - + " GraphLogicalSingleMatch(input=[null]," - + " sentence=[GraphLogicalExpand(tableConfig=[{isAll=false, tables=[knows]}]," - + " alias=[y], opt=[OUT])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," - + " alias=[x], opt=[VERTEX])\n" - + "], matchOpt=[INNER])", - after.explain().trim()); - - Assert.assertEquals( - "RecordType(BIGINT id)\n" - + "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)]," - + " properties=[BIGINT id]) x, CHAR(1) name)\n" - + "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)]," - + " properties=[BIGINT id]) x, CHAR(1) name)\n" - + "RecordType(Graph_Schema_Type(labels=[VertexLabel(person)]," - + " properties=[BIGINT id, CHAR(1) name]) x," - + " Graph_Schema_Type(labels=[EdgeLabel(knows, person, person)], properties=[])" - + " y)\n", - rowTypeString(after)); - } -} diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/CallProcedureTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/CallProcedureTest.java index 21d58c06db55..556e9a853d5d 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/CallProcedureTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/CallProcedureTest.java @@ -30,7 +30,7 @@ public void procedure_1_test() { Utils.evalLogicalPlan("Call ldbc_ic2(10l, 20120112l)", "config/modern/graph.yaml"); Assert.assertEquals("ldbc_ic2(10:BIGINT, 20120112:BIGINT)", logicalPlan.explain().trim()); Assert.assertEquals( - "RecordType(CHAR(1) name)", logicalPlan.getProcedureCall().getType().toString()); + "RecordType(VARCHAR name)", logicalPlan.getProcedureCall().getType().toString()); } // test procedure with invalid parameter types diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java index 6d8f49b7be03..d3c84338cea9 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java @@ -263,7 +263,7 @@ public void match_9_test() { Utils.evalLogicalPlan( "Match (n:person {name: $name}) Where n.age = $age Return n.id;"); Assert.assertEquals( - "[Parameter{name='name', dataType=CHAR(1)}, Parameter{name='age'," + "[Parameter{name='name', dataType=VARCHAR}, Parameter{name='age'," + " dataType=INTEGER}]", plan.getDynamicParams().toString()); Assert.assertEquals("RecordType(BIGINT id)", plan.getOutputType().toString()); @@ -285,10 +285,10 @@ public void match_11_test() { "Match (n:person {name: $name, age: $age}) Where n.id > 10 Return n.id," + " n.name;"); Assert.assertEquals( - "[Parameter{name='name', dataType=CHAR(1)}, Parameter{name='age'," + "[Parameter{name='name', dataType=VARCHAR}, Parameter{name='age'," + " dataType=INTEGER}]", plan.getDynamicParams().toString()); - Assert.assertEquals("RecordType(BIGINT id, CHAR(1) name)", plan.getOutputType().toString()); + Assert.assertEquals("RecordType(BIGINT id, VARCHAR name)", plan.getOutputType().toString()); } @Test diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WhereTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WhereTest.java index 3c1ae5115f7d..4760921128fe 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WhereTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WhereTest.java @@ -266,7 +266,7 @@ public void where_10_test() { .contains( "invalid property lookup operation, cannot get property or" + " extract interval from expr=[creationDate," - + " type=CHAR(1)]")); + + " type=VARCHAR]")); return; } Assert.fail("should have thrown exceptions for property 'name' is not a date type"); diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WithTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WithTest.java index 53fabb714151..c74b12666134 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WithTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/WithTest.java @@ -234,7 +234,7 @@ public void with_13_test() { + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}]," + " alias=[a], opt=[VERTEX])", project.explain().trim()); - Assert.assertEquals("RecordType(CHAR(1) name)", project.getRowType().toString()); + Assert.assertEquals("RecordType(VARCHAR name)", project.getRowType().toString()); } @Test diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java index 97279d78a2da..dd11136a6071 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4x/GraphBuilderTest.java @@ -1755,11 +1755,11 @@ public void g_V_select_a_b_valueMap() { "g.V().hasLabel('person').as('a').out('knows').as('b').select('a'," + " 'b').by(valueMap())"); Assert.assertEquals( - "({_UTF-8'a'=, _UTF-8'id'=, _UTF-8'id'=, _UTF-8'age'=}) MAP>, _UTF-8'b'=, _UTF-8'id'=," + + " ({_UTF-8'name'=, _UTF-8'id'=," + " _UTF-8'creationDate'=, _UTF-8'age'=," - + " _UTF-8'lang'=}) MAP>}) MAP", + + " _UTF-8'lang'=}) MAP>}) MAP", rel.getRowType().getFieldList().get(0).getType().toString()); } diff --git a/interactive_engine/compiler/src/test/resources/config/modern/graph.yaml b/interactive_engine/compiler/src/test/resources/config/modern/graph.yaml index a740b08c2098..0c012174cece 100644 --- a/interactive_engine/compiler/src/test/resources/config/modern/graph.yaml +++ b/interactive_engine/compiler/src/test/resources/config/modern/graph.yaml @@ -33,14 +33,17 @@ stored_procedures: type: string: long_text: + allow_cast: true - name: direction type: string: long_text: + allow_cast: true - name: property type: string: long_text: + allow_cast: true - name: iterations type: primitive_type: DT_SIGNED_INT32 diff --git a/interactive_engine/compiler/src/test/resources/proto/filter_test.json b/interactive_engine/compiler/src/test/resources/proto/filter_test.json index b899f1e5bd74..7975b062cd59 100644 --- a/interactive_engine/compiler/src/test/resources/proto/filter_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/filter_test.json @@ -46,7 +46,8 @@ "nodeType": { "dataType": { "string": { - "longText": { + "char": { + "fixedLength": 5 } } } diff --git a/interactive_engine/compiler/src/test/resources/proto/name_not_containing_expr.json b/interactive_engine/compiler/src/test/resources/proto/name_not_containing_expr.json index d5ff0a449601..5bb9831cdae6 100644 --- a/interactive_engine/compiler/src/test/resources/proto/name_not_containing_expr.json +++ b/interactive_engine/compiler/src/test/resources/proto/name_not_containing_expr.json @@ -46,7 +46,8 @@ "nodeType": { "dataType": { "string": { - "longText": { + "char": { + "fixedLength": 7 } } } @@ -54,4 +55,4 @@ }, { "brace": "RIGHT_BRACE" }] -} +} \ No newline at end of file diff --git a/interactive_engine/compiler/src/test/resources/schema/type_test.yaml b/interactive_engine/compiler/src/test/resources/schema/type_test.yaml new file mode 100644 index 000000000000..1c1a7f6d9aaa --- /dev/null +++ b/interactive_engine/compiler/src/test/resources/schema/type_test.yaml @@ -0,0 +1,37 @@ +name: type_test # then must have a modern dir under ${data} directory +schema: + vertex_types: + - type_name: person + type_id: 0 + x_csr_params: + max_vertex_num: 100 + properties: + - property_id: 0 + property_name: id + property_type: + primitive_type: DT_SIGNED_INT64 + - property_id: 1 + property_name: name1 + property_type: + string: + long_text: + - property_id: 2 + property_name: name2 + property_type: + string: + char: + fixed_length: 255 + - property_id: 3 + property_name: name3 + property_type: + string: + var_char: + max_length: 255 + - property_id: 4 + property_name: value1 + property_type: + decimal: + precision: 4 + scale: 2 + primary_keys: + - id \ No newline at end of file From 6d71da7d298108f2008563b475c0b7cbdb3704e5 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Mon, 30 Dec 2024 16:49:45 +0800 Subject: [PATCH 10/23] minor fix --- .../org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java index 685e62937684..dfd996990633 100644 --- a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java +++ b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandMetaDataImpl.java @@ -64,6 +64,10 @@ public GraphOperandMetaDataImpl( @Override protected Collection getAllowedTypeNames( RelDataTypeFactory typeFactory, SqlTypeFamily family, int iFormalOperand) { + // by default, the types in the same type family are not allowed to cast among each other, + // i.e. int32 to int64, char to varchar + // set the allowCast to true to enable the cast, see + // 'test/resources/config/modern/graph.yaml' for more usage boolean allowCast = this.allowCast.test(iFormalOperand); if (allowCast) { return family.getTypeNames(); From 6fd919fa1019562bffe53d8e362f31b7c24b6009 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Mon, 30 Dec 2024 16:55:54 +0800 Subject: [PATCH 11/23] minor fix --- .../graphscope/common/ir/meta/schema/GSDataTypeDesc.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeDesc.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeDesc.java index 7d42b29205b4..a345689506cf 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeDesc.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GSDataTypeDesc.java @@ -24,6 +24,9 @@ import java.util.Map; +/** + * flex type specification + */ public class GSDataTypeDesc { // support more format of GSDataTypeDesc, i.e. JSON, proto, etc. private final Map yamlDesc; From efeafa2b23226295233e117828ddb41272436f1b Mon Sep 17 00:00:00 2001 From: shirly121 Date: Mon, 30 Dec 2024 17:59:04 +0800 Subject: [PATCH 12/23] fix gaia ci workflow --- .github/workflows/gaia.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/gaia.yml b/.github/workflows/gaia.yml index 6007d9e37a87..bc135601dfb6 100644 --- a/.github/workflows/gaia.yml +++ b/.github/workflows/gaia.yml @@ -52,6 +52,12 @@ jobs: ~/.cache/sccache key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.81.0 + override: true + - name: Rust Format Check run: | cd ${GITHUB_WORKSPACE}/interactive_engine/executor && ./check_format.sh From eb3156105ed31b2b17093deb70a9d64f2a234375 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 1 Jan 2025 16:06:18 +0800 Subject: [PATCH 13/23] fix array max length --- .../common/ir/meta/schema/IrDataTypeConvertor.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java index b6d079786e33..6bb076ff453e 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java @@ -386,7 +386,10 @@ public GSDataTypeDesc convert(RelDataType from) { "component_type", componentType, "max_length", - Integer.MAX_VALUE)); + from.getPrecision() + == RelDataType.PRECISION_NOT_SPECIFIED + ? Integer.MAX_VALUE + : from.getPrecision())); break; case MAP: Map keyType; From 88a7e222bb96a2531f50d2e4a7ebb3d58e1d161a Mon Sep 17 00:00:00 2001 From: "xiaolei.zl@alibaba-inc.com" Date: Thu, 2 Jan 2025 16:15:29 +0800 Subject: [PATCH 14/23] fixing interactive ci Committed-by: xiaolei.zl@alibaba-inc.com from Dev container --- flex/storages/metadata/graph_meta_store.cc | 57 ++++++++++++++++------ flex/storages/metadata/graph_meta_store.h | 1 + 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/flex/storages/metadata/graph_meta_store.cc b/flex/storages/metadata/graph_meta_store.cc index 10b9f079782f..ba453cc955a0 100644 --- a/flex/storages/metadata/graph_meta_store.cc +++ b/flex/storages/metadata/graph_meta_store.cc @@ -66,8 +66,8 @@ const std::vector& get_builtin_plugin_metas() { count_vertices.type = "cypher"; count_vertices.creation_time = GetCurrentTimeStamp(); count_vertices.update_time = GetCurrentTimeStamp(); - count_vertices.params.push_back({"labelName", PropertyType::kString}); - count_vertices.returns.push_back({"count", PropertyType::kInt32}); + count_vertices.params.push_back({"labelName", PropertyType::kString, true}); + count_vertices.returns.push_back({"count", PropertyType::kInt32, false}); builtin_plugins.push_back(count_vertices); // pagerank @@ -80,11 +80,11 @@ const std::vector& get_builtin_plugin_metas() { pagerank.type = "cypher"; pagerank.creation_time = GetCurrentTimeStamp(); pagerank.update_time = GetCurrentTimeStamp(); - pagerank.params.push_back({"vertex_label", PropertyType::kString}); - pagerank.params.push_back({"edge_label", PropertyType::kString}); - pagerank.params.push_back({"damping_factor", PropertyType::kDouble}); - pagerank.params.push_back({"max_iterations", PropertyType::kInt32}); - pagerank.params.push_back({"epsilon", PropertyType::kDouble}); + pagerank.params.push_back({"vertex_label", PropertyType::kString, true}); + pagerank.params.push_back({"edge_label", PropertyType::kString, true}); + pagerank.params.push_back({"damping_factor", PropertyType::kDouble, false}); + pagerank.params.push_back({"max_iterations", PropertyType::kInt32, false}); + pagerank.params.push_back({"epsilon", PropertyType::kDouble, false}); pagerank.returns.push_back({"label_name", PropertyType::kString}); pagerank.returns.push_back({"vertex_oid", PropertyType::kInt64}); pagerank.returns.push_back({"pagerank", PropertyType::kDouble}); @@ -100,9 +100,9 @@ const std::vector& get_builtin_plugin_metas() { k_neighbors.type = "cypher"; k_neighbors.creation_time = GetCurrentTimeStamp(); k_neighbors.update_time = GetCurrentTimeStamp(); - k_neighbors.params.push_back({"label_name", PropertyType::kString}); - k_neighbors.params.push_back({"oid", PropertyType::kInt64}); - k_neighbors.params.push_back({"k", PropertyType::kInt32}); + k_neighbors.params.push_back({"label_name", PropertyType::kString, true}); + k_neighbors.params.push_back({"oid", PropertyType::kInt64, false}); + k_neighbors.params.push_back({"k", PropertyType::kInt32, false}); k_neighbors.returns.push_back({"label_name", PropertyType::kString}); k_neighbors.returns.push_back({"vertex_oid", PropertyType::kInt64}); builtin_plugins.push_back(k_neighbors); @@ -119,14 +119,14 @@ const std::vector& get_builtin_plugin_metas() { shortest_path_among_three.creation_time = GetCurrentTimeStamp(); shortest_path_among_three.update_time = GetCurrentTimeStamp(); shortest_path_among_three.params.push_back( - {"label_name1", PropertyType::kString}); - shortest_path_among_three.params.push_back({"oid1", PropertyType::kInt64}); + {"label_name1", PropertyType::kString, true}); + shortest_path_among_three.params.push_back({"oid1", PropertyType::kInt64, false}); shortest_path_among_three.params.push_back( - {"label_name2", PropertyType::kString}); - shortest_path_among_three.params.push_back({"oid2", PropertyType::kInt64}); + {"label_name2", PropertyType::kString, true}); + shortest_path_among_three.params.push_back({"oid2", PropertyType::kInt64, false}); shortest_path_among_three.params.push_back( - {"label_name3", PropertyType::kString}); - shortest_path_among_three.params.push_back({"oid3", PropertyType::kInt64}); + {"label_name3", PropertyType::kString, true}); + shortest_path_among_three.params.push_back({"oid3", PropertyType::kInt64, false}); shortest_path_among_three.returns.push_back( {"shortest_path_among_three (label name, vertex oid)", PropertyType::kString}); @@ -153,6 +153,7 @@ std::string Parameter::ToJson() const { json.AddMember("name", name, json.GetAllocator()); json.AddMember("type", to_json(type, &json.GetAllocator()), json.GetAllocator()); + json.AddMember("allow_cast", allow_cast, json.GetAllocator()); return rapidjson_stringify(json); } @@ -329,6 +330,7 @@ void PluginMeta::ToJson(rapidjson::Value& json, rapidjson::Document tempDoc(rapidjson::kObjectType, &allocator); tempDoc.AddMember("name", param.name, allocator); tempDoc.AddMember("type", to_json(param.type, &allocator), allocator); + tempDoc.AddMember("allow_cast", param.allow_cast, allocator); params_json.PushBack(tempDoc, allocator); } json.AddMember("params", params_json, allocator); @@ -337,6 +339,7 @@ void PluginMeta::ToJson(rapidjson::Value& json, rapidjson::Document tempDoc(rapidjson::kObjectType, &allocator); tempDoc.AddMember("name", ret.name, allocator); tempDoc.AddMember("type", to_json(ret.type, &allocator), allocator); + tempDoc.AddMember("allow_cast", ret.allow_cast, allocator); returns_json.PushBack(tempDoc, allocator); } json.AddMember("returns", returns_json, allocator); @@ -370,6 +373,11 @@ void PluginMeta::setParamsFromJsonString(const rapidjson::Value& document) { Parameter p; p.name = param["name"].GetString(); p.type = from_json(param["type"]); + if (param.HasMember("allow_cast")) { + p.allow_cast = param["allow_cast"].GetBool(); + } else { + p.allow_cast = false; + } params.push_back(p); } } else { @@ -384,6 +392,11 @@ void PluginMeta::setReturnsFromJsonString(const rapidjson::Value& value) { Parameter p; p.name = ret["name"].GetString(); p.type = from_json(ret["type"]); + if (ret.HasMember("allow_cast")) { + p.allow_cast = ret["allow_cast"].GetBool(); + } else { + p.allow_cast = false; + } returns.push_back(p); } } else { @@ -814,6 +827,9 @@ CreatePluginMetaRequest CreatePluginMetaRequest::FromJson( Parameter p; p.name = param["name"].GetString(); from_json(param["type"], p.type); + if (param.HasMember("allow_cast")) { + p.allow_cast = param["allow_cast"].GetBool(); + } request.params.push_back(p); } } @@ -822,6 +838,9 @@ CreatePluginMetaRequest CreatePluginMetaRequest::FromJson( Parameter p; p.name = ret["name"].GetString(); from_json(ret["type"], p.type); + if (ret.HasMember("allow_cast")) { + p.allow_cast = ret["allow_cast"].GetBool(); + } request.returns.push_back(p); } } @@ -878,6 +897,9 @@ UpdatePluginMetaRequest UpdatePluginMetaRequest::FromJson( Parameter p; p.name = param["name"].GetString(); p.type = from_json(param["type"]); + if (param.HasMember("allow_cast")) { + p.allow_cast = param["allow_cast"].GetBool(); + } request.params->emplace_back(std::move(p)); } } @@ -887,6 +909,9 @@ UpdatePluginMetaRequest UpdatePluginMetaRequest::FromJson( Parameter p; p.name = ret["name"].GetString(); p.type = from_json(ret["type"]); + if (ret.HasMember("allow_cast")) { + p.allow_cast = ret["allow_cast"].GetBool(); + } request.returns->emplace_back(std::move(p)); } } diff --git a/flex/storages/metadata/graph_meta_store.h b/flex/storages/metadata/graph_meta_store.h index 13932c99962e..61c19de36380 100644 --- a/flex/storages/metadata/graph_meta_store.h +++ b/flex/storages/metadata/graph_meta_store.h @@ -39,6 +39,7 @@ using JobId = std::string; struct Parameter { std::string name; PropertyType type; + bool allow_cast{false}; std::string ToJson() const; }; From b0b64caf54df1765d0792e11b4f3902afb75c2e3 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl@alibaba-inc.com" Date: Thu, 2 Jan 2025 17:47:46 +0800 Subject: [PATCH 15/23] format Committed-by: xiaolei.zl@alibaba-inc.com from Dev container --- flex/storages/metadata/graph_meta_store.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/flex/storages/metadata/graph_meta_store.cc b/flex/storages/metadata/graph_meta_store.cc index ba453cc955a0..d8510af46b2f 100644 --- a/flex/storages/metadata/graph_meta_store.cc +++ b/flex/storages/metadata/graph_meta_store.cc @@ -120,13 +120,16 @@ const std::vector& get_builtin_plugin_metas() { shortest_path_among_three.update_time = GetCurrentTimeStamp(); shortest_path_among_three.params.push_back( {"label_name1", PropertyType::kString, true}); - shortest_path_among_three.params.push_back({"oid1", PropertyType::kInt64, false}); + shortest_path_among_three.params.push_back( + {"oid1", PropertyType::kInt64, false}); shortest_path_among_three.params.push_back( {"label_name2", PropertyType::kString, true}); - shortest_path_among_three.params.push_back({"oid2", PropertyType::kInt64, false}); + shortest_path_among_three.params.push_back( + {"oid2", PropertyType::kInt64, false}); shortest_path_among_three.params.push_back( {"label_name3", PropertyType::kString, true}); - shortest_path_among_three.params.push_back({"oid3", PropertyType::kInt64, false}); + shortest_path_among_three.params.push_back( + {"oid3", PropertyType::kInt64, false}); shortest_path_among_three.returns.push_back( {"shortest_path_among_three (label name, vertex oid)", PropertyType::kString}); From cb385e0acb3294c16c0b38cb99264d6b30551f12 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl@alibaba-inc.com" Date: Fri, 3 Jan 2025 08:23:47 +0800 Subject: [PATCH 16/23] add allow_cast in openapi spec Committed-by: xiaolei.zl@alibaba-inc.com from Dev container --- .github/workflows/interactive.yml | 3 --- flex/openapi/openapi_coordinator.yaml | 2 ++ flex/openapi/openapi_interactive.yaml | 3 +++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 716c79ac2627..d160d4636b60 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -212,12 +212,9 @@ jobs: GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/ cd ${GITHUB_WORKSPACE}/flex/tests/hqps sed -i 's/interactive_workspace/temp_workspace/g' ./interactive_config_test.yaml - # set thread_num_per_worker to 4 - sed -i 's/thread_num_per_worker: 1/thread_num_per_worker: 4/g' ./interactive_config_test.yaml bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml java bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml python sed -i 's/temp_workspace/interactive_workspace/g' ./interactive_config_test.yaml - sed -i 's/thread_num_per_worker: 4/thread_num_per_worker: 1/g' ./interactive_config_test.yaml - name: Robustness test env: diff --git a/flex/openapi/openapi_coordinator.yaml b/flex/openapi/openapi_coordinator.yaml index a3029de3c5cc..88d5bb9affd1 100644 --- a/flex/openapi/openapi_coordinator.yaml +++ b/flex/openapi/openapi_coordinator.yaml @@ -331,6 +331,8 @@ components: type: string type: $ref: '#/components/schemas/GSDataType' + allow_cast: + type: boolean BasePropertyMeta: required: diff --git a/flex/openapi/openapi_interactive.yaml b/flex/openapi/openapi_interactive.yaml index 961a904869db..bdbf4474907b 100644 --- a/flex/openapi/openapi_interactive.yaml +++ b/flex/openapi/openapi_interactive.yaml @@ -1436,6 +1436,9 @@ components: example: param1 type: $ref: '#/components/schemas/GSDataType' + allow_cast: + type: boolean + example: true VertexRequest: x-body-name: vertex_request type: object From 3e76cc464fa4b2e4f3057a65b54a228a5e8e4c14 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Mon, 6 Jan 2025 10:45:20 +0800 Subject: [PATCH 17/23] loose type checking for string --- .../org/apache/calcite/sql/type/GraphOperandTypes.java | 9 ++++++++- .../compiler/src/test/resources/config/modern/graph.yaml | 3 --- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java index 7d5c4533a955..283954e84491 100644 --- a/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java +++ b/interactive_engine/compiler/src/main/java/org/apache/calcite/sql/type/GraphOperandTypes.java @@ -106,6 +106,13 @@ public static SqlOperandTypeChecker metaTypeChecker(StoredProcedureMeta meta) { parameters.stream().map(p -> p.getDataType()).collect(Collectors.toList()), i -> parameters.get(i).getName(), i -> false, - i -> parameters.get(i).allowCast()); + i -> { + boolean allowCast = parameters.get(i).allowCast(); + if (allowCast) return true; + // loose the type checking for string type + SqlTypeFamily typeFamily = + parameters.get(i).getDataType().getSqlTypeName().getFamily(); + return typeFamily == SqlTypeFamily.CHARACTER; + }); } } diff --git a/interactive_engine/compiler/src/test/resources/config/modern/graph.yaml b/interactive_engine/compiler/src/test/resources/config/modern/graph.yaml index 0c012174cece..a740b08c2098 100644 --- a/interactive_engine/compiler/src/test/resources/config/modern/graph.yaml +++ b/interactive_engine/compiler/src/test/resources/config/modern/graph.yaml @@ -33,17 +33,14 @@ stored_procedures: type: string: long_text: - allow_cast: true - name: direction type: string: long_text: - allow_cast: true - name: property type: string: long_text: - allow_cast: true - name: iterations type: primitive_type: DT_SIGNED_INT32 From e1f8dd8aba843431fd2910ac4290cdbe2eb7a8c8 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Mon, 6 Jan 2025 10:53:05 +0800 Subject: [PATCH 18/23] fix type conversion in v6d_ffi to the new flex datatype --- .../src/store_impl/v6d/read_ffi.rs | 210 ++++++++++++++---- 1 file changed, 170 insertions(+), 40 deletions(-) diff --git a/interactive_engine/executor/store/global_query/src/store_impl/v6d/read_ffi.rs b/interactive_engine/executor/store/global_query/src/store_impl/v6d/read_ffi.rs index 0f48a90b2f85..1fc8e0a503ba 100644 --- a/interactive_engine/executor/store/global_query/src/store_impl/v6d/read_ffi.rs +++ b/interactive_engine/executor/store/global_query/src/store_impl/v6d/read_ffi.rs @@ -971,51 +971,181 @@ impl PropertyType { } pub fn from_common_data_type(raw_type: common_pb::DataType) -> Self { - match raw_type { - common_pb::DataType::Boolean => PropertyType::Bool, - common_pb::DataType::Int32 => PropertyType::Int, - common_pb::DataType::Int64 => PropertyType::Long, - common_pb::DataType::Double => PropertyType::Double, - common_pb::DataType::String => PropertyType::String, - common_pb::DataType::Bytes => PropertyType::Bytes, - common_pb::DataType::Int32Array => PropertyType::IntList, - common_pb::DataType::Int64Array => PropertyType::LongList, - common_pb::DataType::DoubleArray => PropertyType::DoubleList, - common_pb::DataType::StringArray => PropertyType::StringList, - common_pb::DataType::Date32 => PropertyType::Date32, - common_pb::DataType::Time32 => PropertyType::Time32MS, - common_pb::DataType::Timestamp => PropertyType::TimestampNS, - _ => { - unimplemented!("Unsupported data type {:?}", raw_type) + if let Some(item) = raw_type.item.as_ref() { + match item { + common_pb::data_type::Item::PrimitiveType(primitive) => { + let primitive_type: common_pb::PrimitiveType = + unsafe { std::mem::transmute(*primitive) }; + match primitive_type { + common_pb::PrimitiveType::DtAny => PropertyType::Bytes, + common_pb::PrimitiveType::DtSignedInt32 => PropertyType::Int, + common_pb::PrimitiveType::DtUnsignedInt32 => PropertyType::Int, + common_pb::PrimitiveType::DtSignedInt64 => PropertyType::Long, + common_pb::PrimitiveType::DtUnsignedInt64 => PropertyType::Long, + common_pb::PrimitiveType::DtBool => PropertyType::Bool, + common_pb::PrimitiveType::DtFloat => PropertyType::Float, + common_pb::PrimitiveType::DtDouble => PropertyType::Double, + } + } + common_pb::data_type::Item::String(string) => match string.item.as_ref() { + Some(common_pb::string::Item::Char(_char)) => { + if _char.fixed_length == 1 { + PropertyType::Char + } else { + PropertyType::String + } + } + _ => PropertyType::String, + }, + common_pb::data_type::Item::Temporal(temporal) => match temporal.item.as_ref() { + Some(common_pb::temporal::Item::Date32(_date32)) => PropertyType::Date32, + Some(common_pb::temporal::Item::Time32(_time32)) => PropertyType::Time32MS, + Some(common_pb::temporal::Item::Timestamp(_timestamp)) => PropertyType::TimestampMS, + _ => { + unimplemented!("Unsupported data type {:?}", temporal) + } + }, + common_pb::data_type::Item::Array(array) => { + if let Some(component_type) = array.component_type.as_ref() { + match component_type.item.as_ref() { + Some(common_pb::data_type::Item::PrimitiveType(primitive)) => { + let primitive_type: common_pb::PrimitiveType = + unsafe { std::mem::transmute(*primitive) }; + match primitive_type { + common_pb::PrimitiveType::DtSignedInt32 + | common_pb::PrimitiveType::DtUnsignedInt32 => PropertyType::IntList, + common_pb::PrimitiveType::DtSignedInt64 + | common_pb::PrimitiveType::DtUnsignedInt64 => PropertyType::LongList, + common_pb::PrimitiveType::DtFloat => PropertyType::FloatList, + common_pb::PrimitiveType::DtDouble => PropertyType::DoubleList, + _ => { + unimplemented!("Unsupported data type {:?}", primitive_type) + } + } + } + Some(common_pb::data_type::Item::String(_string)) => PropertyType::StringList, + _ => { + unimplemented!("Unsupported data type {:?}", component_type) + } + } + } else { + unimplemented!("empty array component_type {:?}", array) + } + } + _ => unimplemented!("Unsupported data type {:?}", item), } + } else { + unimplemented!("empty data type item {:?}", raw_type) } } pub fn to_common_data_type(&self) -> common_pb::DataType { match *self { - PropertyType::Bool => common_pb::DataType::Boolean, - PropertyType::Int => common_pb::DataType::Int32, - PropertyType::Long => common_pb::DataType::Int64, - PropertyType::Double => common_pb::DataType::Double, - PropertyType::String => common_pb::DataType::String, - PropertyType::Bytes => common_pb::DataType::Bytes, - PropertyType::IntList => common_pb::DataType::Int32Array, - PropertyType::LongList => common_pb::DataType::Int64Array, - PropertyType::DoubleList => common_pb::DataType::DoubleArray, - PropertyType::StringList => common_pb::DataType::StringArray, - PropertyType::Date32 => common_pb::DataType::Date32, - PropertyType::Date64 => common_pb::DataType::Timestamp, - PropertyType::Time32S => common_pb::DataType::Time32, - PropertyType::Time32MS => common_pb::DataType::Time32, - PropertyType::Time32US => common_pb::DataType::Time32, - PropertyType::Time32NS => common_pb::DataType::Time32, - PropertyType::Time64S => common_pb::DataType::Time32, - PropertyType::Time64MS => common_pb::DataType::Time32, - PropertyType::Time64US => common_pb::DataType::Time32, - PropertyType::Time64NS => common_pb::DataType::Time32, - PropertyType::TimestampS => common_pb::DataType::Timestamp, - PropertyType::TimestampMS => common_pb::DataType::Timestamp, - PropertyType::TimestampUS => common_pb::DataType::Timestamp, - PropertyType::TimestampNS => common_pb::DataType::Timestamp, + PropertyType::Bool => common_pb::DataType { + item: Some(common_pb::data_type::Item::PrimitiveType( + common_pb::PrimitiveType::DtBool as i32, + )), + }, + PropertyType::Char => common_pb::DataType { + item: Some(common_pb::data_type::Item::String(common_pb::String { + item: Some(common_pb::string::Item::Char(common_pb::string::Char { fixed_length: 1 })), + })), + }, + PropertyType::Int => common_pb::DataType { + item: Some(common_pb::data_type::Item::PrimitiveType( + common_pb::PrimitiveType::DtSignedInt32 as i32, + )), + }, + PropertyType::Long => common_pb::DataType { + item: Some(common_pb::data_type::Item::PrimitiveType( + common_pb::PrimitiveType::DtSignedInt64 as i32, + )), + }, + PropertyType::Float => common_pb::DataType { + item: Some(common_pb::data_type::Item::PrimitiveType( + common_pb::PrimitiveType::DtFloat as i32, + )), + }, + PropertyType::Double => common_pb::DataType { + item: Some(common_pb::data_type::Item::PrimitiveType( + common_pb::PrimitiveType::DtDouble as i32, + )), + }, + PropertyType::String => common_pb::DataType { + item: Some(common_pb::data_type::Item::String(common_pb::String { + item: Some(common_pb::string::Item::LongText(common_pb::string::LongText {})), + })), + }, + PropertyType::Bytes => common_pb::DataType { + item: Some(common_pb::data_type::Item::PrimitiveType( + common_pb::PrimitiveType::DtAny as i32, + )), + }, + PropertyType::IntList => common_pb::DataType { + item: Some(common_pb::data_type::Item::Array(Box::new(common_pb::Array { + component_type: Some(Box::new(common_pb::DataType { + item: Some(common_pb::data_type::Item::PrimitiveType( + common_pb::PrimitiveType::DtSignedInt32 as i32, + )), + })), + // max_length is not used + max_length: 1024, + }))), + }, + PropertyType::LongList => common_pb::DataType { + item: Some(common_pb::data_type::Item::Array(Box::new(common_pb::Array { + component_type: Some(Box::new(common_pb::DataType { + item: Some(common_pb::data_type::Item::PrimitiveType( + common_pb::PrimitiveType::DtSignedInt64 as i32, + )), + })), + max_length: 1024, + }))), + }, + PropertyType::FloatList => common_pb::DataType { + item: Some(common_pb::data_type::Item::Array(Box::new(common_pb::Array { + component_type: Some(Box::new(common_pb::DataType { + item: Some(common_pb::data_type::Item::PrimitiveType( + common_pb::PrimitiveType::DtFloat as i32, + )), + })), + max_length: 1024, + }))), + }, + PropertyType::DoubleList => common_pb::DataType { + item: Some(common_pb::data_type::Item::Array(Box::new(common_pb::Array { + component_type: Some(Box::new(common_pb::DataType { + item: Some(common_pb::data_type::Item::PrimitiveType( + common_pb::PrimitiveType::DtDouble as i32, + )), + })), + max_length: 1024, + }))), + }, + PropertyType::StringList => common_pb::DataType { + item: Some(common_pb::data_type::Item::Array(Box::new(common_pb::Array { + component_type: Some(Box::new(common_pb::DataType { + item: Some(common_pb::data_type::Item::String(common_pb::String { + item: Some(common_pb::string::Item::LongText(common_pb::string::LongText {})), + })), + })), + max_length: 1024, + }))), + }, + PropertyType::Date32 => common_pb::DataType { + item: Some(common_pb::data_type::Item::Temporal(common_pb::Temporal { + item: Some(common_pb::temporal::Item::Date32(common_pb::temporal::Date32 {})), + })), + }, + PropertyType::Time32MS => common_pb::DataType { + item: Some(common_pb::data_type::Item::Temporal(common_pb::Temporal { + item: Some(common_pb::temporal::Item::Time32(common_pb::temporal::Time32 {})), + })), + }, + PropertyType::TimestampMS => common_pb::DataType { + item: Some(common_pb::data_type::Item::Temporal(common_pb::Temporal { + item: Some(common_pb::temporal::Item::Timestamp(common_pb::temporal::Timestamp {})), + })), + }, _ => { unimplemented!("Unsupported data type {:?}", *self) } From bb959b4447230c9bfe9962a9aa598bf17f151a07 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Mon, 6 Jan 2025 16:46:49 +0800 Subject: [PATCH 19/23] add null type --- .../common/ir/meta/schema/IrDataTypeConvertor.java | 5 +++++ interactive_engine/executor/ir/proto/basic_type.proto | 1 + 2 files changed, 6 insertions(+) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java index 6bb076ff453e..9cd921e6c975 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrDataTypeConvertor.java @@ -198,6 +198,8 @@ public RelDataType convert(GSDataTypeDesc from) { Object value; if ((value = typeMap.get("primitive_type")) != null) { switch (value.toString()) { + case "DT_NULL": + return typeFactory.createSqlType(SqlTypeName.NULL); case "DT_ANY": // any type return typeFactory.createSqlType(SqlTypeName.ANY); @@ -324,6 +326,9 @@ public GSDataTypeDesc convert(RelDataType from) { SqlTypeName typeName = from.getSqlTypeName(); Map yamlDesc; switch (typeName) { + case NULL: + yamlDesc = ImmutableMap.of("primitive_type", "DT_NULL"); + break; case INTEGER: yamlDesc = ImmutableMap.of("primitive_type", "DT_SIGNED_INT32"); break; diff --git a/interactive_engine/executor/ir/proto/basic_type.proto b/interactive_engine/executor/ir/proto/basic_type.proto index 8e1d72d45c24..849711f05ff0 100644 --- a/interactive_engine/executor/ir/proto/basic_type.proto +++ b/interactive_engine/executor/ir/proto/basic_type.proto @@ -13,6 +13,7 @@ enum PrimitiveType { DT_BOOL = 5; DT_FLOAT = 6; DT_DOUBLE = 7; + DT_NULL = 8; // denote the type of 'null' value } message Decimal { // precision=4 scale=2 : 23.12 From 82395952238eaacfcb6ac88b5e629e70f4b6d759 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl@alibaba-inc.com" Date: Mon, 6 Jan 2025 16:16:58 +0800 Subject: [PATCH 20/23] fixing ci Committed-by: xiaolei.zl@alibaba-inc.com from Dev container Committed-by: xiaolei.zl@alibaba-inc.com from Dev container --- flex/codegen/src/graph_types.h | 18 +++++++++++++++++- .../graph_db/runtime/adhoc/expr_impl.cc | 6 +++++- flex/engines/graph_db/runtime/common/rt_any.cc | 2 ++ flex/tests/hqps/hqps_codegen_test.sh | 3 ++- .../cypher/result/CypherRecordParser.java | 1 + 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/flex/codegen/src/graph_types.h b/flex/codegen/src/graph_types.h index c9e45c2e2df5..c38affc93127 100644 --- a/flex/codegen/src/graph_types.h +++ b/flex/codegen/src/graph_types.h @@ -85,6 +85,22 @@ static codegen::DataType primitive_type_to_data_type( } } +static codegen::DataType temporal_type_to_data_type( + const common::Temporal& type) { + switch (type.item_case()) { + case common::Temporal::ItemCase::kDate: + return codegen::DataType::kDate; + case common::Temporal::ItemCase::kTime: + return codegen::DataType::kTime; + case common::Temporal::kTimestamp: + return codegen::DataType::kTimeStamp; + default: + throw std::runtime_error( + "unknown temporal type when converting temporal type to data type:" + + std::to_string(static_cast(type.item_case()))); + } +} + static codegen::DataType common_data_type_pb_2_data_type( const common::DataType& data_type) { switch (data_type.item_case()) { @@ -95,7 +111,7 @@ static codegen::DataType common_data_type_pb_2_data_type( case common::DataType::ItemCase::kString: return codegen::DataType::kString; case common::DataType::ItemCase::kTemporal: - LOG(FATAL) << "Not support temporal type"; + return temporal_type_to_data_type(data_type.temporal()); case common::DataType::ItemCase::kArray: case common::DataType::ItemCase::kMap: LOG(FATAL) << "Not support array or map type"; diff --git a/flex/engines/graph_db/runtime/adhoc/expr_impl.cc b/flex/engines/graph_db/runtime/adhoc/expr_impl.cc index 50618aebdf04..e72f003518b2 100644 --- a/flex/engines/graph_db/runtime/adhoc/expr_impl.cc +++ b/flex/engines/graph_db/runtime/adhoc/expr_impl.cc @@ -466,7 +466,11 @@ static RTAny parse_param(const common::DynamicParam& param, if (dt.temporal().item_case() == common::Temporal::kDate32) { int64_t val = std::stoll(input.at(name)); return RTAny::from_int64(val); - } else { + } else if (dt.temporal().item_case() == common::Temporal::kTimestamp) { + int64_t val = std::stoll(input.at(name)); + return RTAny::from_int64(val); + } + else { LOG(FATAL) << "not support type: " << dt.temporal().DebugString(); } } else if (dt.item_case() == common::DataType::ItemCase::kString) { diff --git a/flex/engines/graph_db/runtime/common/rt_any.cc b/flex/engines/graph_db/runtime/common/rt_any.cc index 1908c1ccd084..ea447f12443c 100644 --- a/flex/engines/graph_db/runtime/common/rt_any.cc +++ b/flex/engines/graph_db/runtime/common/rt_any.cc @@ -79,6 +79,8 @@ RTAnyType parse_from_ir_data_type(const ::common::IrDataType& dt) { case ::common::DataType::kTemporal: { if (ddt.temporal().item_case() == ::common::Temporal::kDate32) { return RTAnyType::kDate32; + } else if (ddt.temporal().item_case() == ::common::Temporal::kTimestamp) { + return RTAnyType::kDate32; } else { LOG(FATAL) << "unrecoginized temporal type - " << ddt.temporal().DebugString(); diff --git a/flex/tests/hqps/hqps_codegen_test.sh b/flex/tests/hqps/hqps_codegen_test.sh index a0b460ab2f84..e4b33fc51ce5 100644 --- a/flex/tests/hqps/hqps_codegen_test.sh +++ b/flex/tests/hqps/hqps_codegen_test.sh @@ -75,7 +75,8 @@ test_codegen_on_ldbc_cbo(){ test_codegen_on_ldbc_rbo(){ sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ${RBO_ENGINE_CONFIG_PATH} - for i in 1 2 3 4 5 6 7 8 9 10 11 12; + # ic1_adhoc is skipped since NULL type is not supported now + for i in 2 3 4 5 6 7 8 9 10 11 12; do cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codegen/" cmd=${cmd}" -o=/tmp/plugin --ir_conf=${RBO_ENGINE_CONFIG_PATH} " diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java index d7ccba292a61..fb3f083aedc9 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/result/CypherRecordParser.java @@ -146,6 +146,7 @@ protected AnyValue parseCollection(IrResult.Collection collection, RelDataType c .mapToDouble(k -> k.getObject().getF64()) .toArray()); case CHAR: + case VARCHAR: return Values.stringArray( collection.getCollectionList().stream() .map(k -> k.getObject().getStr()) From a15874c3adc4f41f46a8c0a141ae3d14bc6ae9d5 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl@alibaba-inc.com" Date: Mon, 6 Jan 2025 17:07:02 +0800 Subject: [PATCH 21/23] add back codegen test cases Committed-by: xiaolei.zl@alibaba-inc.com from Dev container --- flex/tests/hqps/hqps_codegen_test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flex/tests/hqps/hqps_codegen_test.sh b/flex/tests/hqps/hqps_codegen_test.sh index e4b33fc51ce5..a0b460ab2f84 100644 --- a/flex/tests/hqps/hqps_codegen_test.sh +++ b/flex/tests/hqps/hqps_codegen_test.sh @@ -75,8 +75,7 @@ test_codegen_on_ldbc_cbo(){ test_codegen_on_ldbc_rbo(){ sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ${RBO_ENGINE_CONFIG_PATH} - # ic1_adhoc is skipped since NULL type is not supported now - for i in 2 3 4 5 6 7 8 9 10 11 12; + for i in 1 2 3 4 5 6 7 8 9 10 11 12; do cmd="${CODEGEN_SCRIPT} -e=hqps -i=${FLEX_HOME}/resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codegen/" cmd=${cmd}" -o=/tmp/plugin --ir_conf=${RBO_ENGINE_CONFIG_PATH} " From e9ea646d33e0fc08db73a1e053109a6ea786be22 Mon Sep 17 00:00:00 2001 From: "bingqing.lbq" Date: Mon, 6 Jan 2025 17:08:23 +0800 Subject: [PATCH 22/23] add datatype in schema.proto back, and fix in conversion from json to proto Committed-by: bingqing.lbq from Dev container --- .../executor/ir/core/src/plan/meta.rs | 93 ++++++++++++++++++- .../executor/ir/proto/schema.proto | 3 +- 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/interactive_engine/executor/ir/core/src/plan/meta.rs b/interactive_engine/executor/ir/core/src/plan/meta.rs index a8fc82e45c98..fcee1717c628 100644 --- a/interactive_engine/executor/ir/core/src/plan/meta.rs +++ b/interactive_engine/executor/ir/core/src/plan/meta.rs @@ -407,9 +407,96 @@ impl JsonIO for Schema { where Self: Sized, { - let schema_pb = serde_json::from_reader::<_, schema_pb::Schema>(reader)?; - let schema = Schema::from(schema_pb); - Ok(schema) + let mut json_value: serde_json::Value = serde_json::from_reader(reader)?; + + // Process the data_type field to convert enum (int32) to new data_type structure + if let Some(obj) = json_value.as_object_mut() { + if let Some(entities) = obj + .get_mut("entities") + .and_then(|v| v.as_array_mut()) + { + for entity in entities { + if let Some(entity_obj) = entity.as_object_mut() { + if let Some(columns) = entity_obj + .get_mut("columns") + .and_then(|v| v.as_array_mut()) + { + for column in columns { + if let Some(column_obj) = column.as_object_mut() { + if let Some(data_type_value) = column_obj.get("data_type") { + if let Some(data_type_int) = data_type_value.as_i64() { + column_obj.insert( + "data_type".to_string(), + convert_data_type(data_type_int), + ); + } + } + } + } + } + } + } + } + + if let Some(relations) = obj + .get_mut("relations") + .and_then(|v| v.as_array_mut()) + { + for relation in relations { + if let Some(relation_obj) = relation.as_object_mut() { + if let Some(columns) = relation_obj + .get_mut("columns") + .and_then(|v| v.as_array_mut()) + { + for column in columns { + if let Some(column_obj) = column.as_object_mut() { + if let Some(data_type_value) = column_obj.get("data_type") { + if let Some(data_type_int) = data_type_value.as_i64() { + column_obj.insert( + "data_type".to_string(), + convert_data_type(data_type_int), + ); + } + } + } + } + } + } + } + } + } + + let schema_pb: schema_pb::Schema = serde_json::from_value(json_value)?; + Ok(Schema::from(schema_pb)) + } +} + +// Helper function to convert int32 to the appropriate DataType structure +fn convert_data_type(data_type_int: i64) -> serde_json::Value { + use serde_json::json; + match data_type_int { + // Primitive types mapping + 0 => json!({ "primitive_type": "DT_BOOL" }), // BOOLEAN + 1 => json!({ "primitive_type": "DT_SIGNED_INT32" }), // INT32 + 2 => json!({ "primitive_type": "DT_SIGNED_INT64" }), // INT64 + 3 => json!({ "primitive_type": "DT_DOUBLE" }), // DOUBLE + + // String type mapping + 4 => json!({ "string": { "long_text": {} } }), // STRING + + // Array types mapping + 6 => json!({ "array": { "component_type": { "primitive_type": "DT_SIGNED_INT32" } } }), // INT32_ARRAY + 7 => json!({ "array": { "component_type": { "primitive_type": "DT_SIGNED_INT64" } } }), // INT64_ARRAY + 8 => json!({ "array": { "component_type": { "primitive_type": "DT_DOUBLE" } } }), // DOUBLE_ARRAY + 9 => json!({ "array": { "component_type": { "string": { "long_text": {} } } } }), // STRING_ARRAY + + // Temporal types mapping + 12 => json!({ "temporal": { "date32": {} } }), // DATE32 + 13 => json!({ "temporal": { "time32": {} } }), // TIME32 + 14 => json!({ "temporal": { "timestamp": {} } }), // TIMESTAMP + + // Other types handling (default to a NONE-like type if applicable) + _ => json!({ "primitive_type": "DT_ANY" }), // NONE or unsupported types } } diff --git a/interactive_engine/executor/ir/proto/schema.proto b/interactive_engine/executor/ir/proto/schema.proto index 22d28ea664c8..6cd7b58ef97a 100644 --- a/interactive_engine/executor/ir/proto/schema.proto +++ b/interactive_engine/executor/ir/proto/schema.proto @@ -30,8 +30,9 @@ message LabelMeta { message ColumnMeta { LabelMeta key = 1; + common.DataType data_type = 2; // Whether this column is a part of a primary key - bool is_primary_key = 2; + bool is_primary_key = 3; } message EntityMeta { From b6238ced6b9d07d4053a994132c004a0a09ed51c Mon Sep 17 00:00:00 2001 From: "bingqing.lbq" Date: Mon, 6 Jan 2025 17:08:45 +0800 Subject: [PATCH 23/23] fix in ffi_writer for subgraph sink Committed-by: bingqing.lbq from Dev container --- .../src/adapters/vineyard_store/write_graph.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/interactive_engine/executor/ir/graph_proxy/src/adapters/vineyard_store/write_graph.rs b/interactive_engine/executor/ir/graph_proxy/src/adapters/vineyard_store/write_graph.rs index 5be6d66de850..5b249e31daa9 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/adapters/vineyard_store/write_graph.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/adapters/vineyard_store/write_graph.rs @@ -17,7 +17,6 @@ use std::ffi::CString; use global_query::store_impl::v6d::read_ffi::*; use global_query::store_impl::v6d::write_ffi::*; -use ir_common::generated::common as common_pb; use ir_common::generated::schema as schema_pb; use ir_common::{KeyId, LabelId, NameOrId, OneOrMany}; @@ -173,9 +172,7 @@ fn build_vineyard_schema(schema: &schema_pb::Schema) -> GraphProxyResult GraphProxyResult