From dc5064baaf64f755012a36621ff2535068c8e9ba Mon Sep 17 00:00:00 2001 From: pragyasri-gs <132895593+pragyasri-gs@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:10:12 +0530 Subject: [PATCH] Support for match function in relational semi-structured queries (#2361) --- .../pure/router/routing/router_routing.pure | 8 +- .../TestSnowflakeSemiStructuredMatching.java | 177 +++++++++ .../semiStructuredMatching.pure | 342 +++++++++++++++++ .../TestSemiStructuredMatching.java | 219 +++++++++++ .../semiStructuredMatching.pure | 348 ++++++++++++++++++ .../pureToSQLQuery/pureToSQLQuery.pure | 68 ++++ 6 files changed, 1158 insertions(+), 4 deletions(-) create mode 100644 legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSnowflakeSemiStructuredMatching.java create mode 100644 legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredMatching.pure create mode 100644 legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSemiStructuredMatching.java create mode 100644 legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredMatching.pure diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure index 9e1c0747789..80a447e1d3d 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure @@ -132,11 +132,11 @@ function <> meta::pure::router::routing::routeValueSpecification |$state, |let processedValues = processCollection($state, $i.values->evaluateAndDeactivate(), $executionContext, $vars, $inScopeVars, v:Any[1]|true, $extensions, $debug); let last = $processedValues->last()->toOne(); - let shouldClean = $processedValues.value->evaluateAndDeactivate()->forAll(p|$p->instanceOf(InstanceValue) && $p->cast(@InstanceValue).genericType.rawType != LambdaFunction); - + let shouldClean = $processedValues->evaluateAndDeactivate().value->forAll(p|$p->instanceOf(InstanceValue) && $p->cast(@InstanceValue).genericType.rawType != LambdaFunction); + ^$last(value = if ($shouldClean, - | ^$i(values = $processedValues.value->evaluateAndDeactivate()->cast(@InstanceValue)->map(e|$e.values)), - | ^$i(values = $processedValues.value))); + | ^$i(values = $processedValues->evaluateAndDeactivate().value->cast(@InstanceValue)->map(e|$e.values)), + | ^$i(values = $processedValues->evaluateAndDeactivate().value))); ); );, cs:ClassSetImplementationHolder[1] | let param = $cs.value->cast(@InstanceValue).values->at(0)->cast(@Class); // TODO: cleanup needed diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSnowflakeSemiStructuredMatching.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSnowflakeSemiStructuredMatching.java new file mode 100644 index 00000000000..93645d33e31 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSnowflakeSemiStructuredMatching.java @@ -0,0 +1,177 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.plan.execution.stores.relational.test.semiStructured; + +import org.eclipse.collections.impl.multimap.set.SynchronizedSetMultimap; +import org.junit.Assert; +import org.junit.Test; + +public class TestSnowflakeSemiStructuredMatching extends AbstractTestSnowflakeSemiStructured +{ + private static final String snowflakeMapping = "match::mapping::SnowflakeMapping"; + private static final String snowflakeRuntime = "match::runtime::SnowflakeRuntime"; + + @Test + public void testSemiStructuredMatchComplexProperty() + { + String queryFunction = "match::semiStructuredMatchComplexProperty__TabularDataSet_1_"; + String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime); + String snowflakeExpected = + " Relational\n" + + " (\n" + + " type = TDS[(Customer Address, String, \"\", \"\")]\n" + + " resultColumns = [(\"Customer Address\", \"\")]\n" + + " sql = select case when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('BillingAddress') then \"root\".CUSTOMER['customerAddress']['billAddress']::varchar when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('ShippingAddress') then \"root\".CUSTOMER['customerAddress']['shipAddress']::varchar else 'Default Address' end as \"Customer Address\" from ORDER_SCHEMA.ORDER_TABLE as \"root\"\n" + + " connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" + + " )\n"; + String TDSType = " type = TDS[(Customer Address, String, \"\", \"\")]\n"; + Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan); + } + + + @Test + public void testSemiStructuredMatchWithMultipleProject() + { + String queryFunction = "match::semiStructuredMatchWithMultipleProject__TabularDataSet_1_"; + String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime); + String snowflakeExpected = + " Relational\n" + + " (\n" + + " type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n" + + " resultColumns = [(\"Customer Address\", \"\"), (\"Order Price\", \"\")]\n" + + " sql = select case when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('BillingAddress') then \"root\".CUSTOMER['customerAddress']['billAddress']::varchar when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('ShippingAddress') then \"root\".CUSTOMER['customerAddress']['shipAddress']::varchar else null end as \"Customer Address\", case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CashOnDeliveryPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountToBePaid'] when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountPaid'] else null end as \"Order Price\" from ORDER_SCHEMA.ORDER_TABLE as \"root\"\n" + + " connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" + + " )\n"; + String TDSType = " type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n"; + Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan); + } + + @Test + public void testSemiStructuredMatchWithComplexFilter() + { + String queryFunction = "match::semiStructuredMatchWithComplexFilter__TabularDataSet_1_"; + String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime); + String snowflakeExpected = + " Relational\n" + + " (\n" + + " type = TDS[(Customer Address, String, \"\", \"\")]\n" + + " resultColumns = [(\"Customer Address\", \"\")]\n" + + " sql = select case when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('BillingAddress') then \"root\".CUSTOMER['customerAddress']['billAddress']::varchar when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('ShippingAddress') then \"root\".CUSTOMER['customerAddress']['shipAddress']::varchar else null end as \"Customer Address\" from ORDER_SCHEMA.ORDER_TABLE as \"root\" where case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CashOnDeliveryPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountToBePaid'] when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountPaid'] else null end < 200\n" + + " connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" + + " )\n"; + String TDSType = " type = TDS[(Customer Address, String, \"\", \"\")]\n"; + Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan); + } + + + @Test + public void testSemiStructuredMatchWithVariableAccess() + { + String queryFunction = "match::semiStructuredMatchWithVariableAccess__TabularDataSet_1_"; + String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime); + String snowflakeExpected = "Sequence\n" + + "(\n" + + " type = TDS[(Max Amount Flag, Boolean, \"\", \"\")]\n" + + " (\n" + + " Allocation\n" + + " (\n" + + " type = Integer\n" + + " resultSizeRange = 1\n" + + " name = maxAmount\n" + + " value = \n" + + " (\n" + + " Constant\n" + + " (\n" + + " type = Integer\n" + + " resultSizeRange = 1\n" + + " values=[200]\n" + + " )\n" + + " )\n" + + " )\n" + + " RelationalBlockExecutionNode\n" + + " (\n" + + " type = TDS[(Max Amount Flag, Boolean, \"\", \"\")]\n" + + " (\n" + + " SQL\n" + + " (\n" + + " type = Void\n" + + " resultColumns = []\n" + + " sql = ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"${execID}\", \"engineUser\" : \"${userId}\", \"referer\" : \"${referer}\"}';\n" + + " connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" + + " )\n" + + " Relational\n" + + " (\n" + + " type = TDS[(Max Amount Flag, Boolean, \"\", \"\")]\n" + + " resultColumns = [(\"Max Amount Flag\", \"\")]\n" + + " sql = select case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CashOnDeliveryPayment') then case when \"root\".CUSTOMER['transactionDetails']['payment']['amountToBePaid'] < ${maxAmount} then 'true' else 'false' end when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then case when \"root\".CUSTOMER['transactionDetails']['payment']['amountPaid'] < ${maxAmount} then 'true' else 'false' end else null end as \"Max Amount Flag\" from ORDER_SCHEMA.ORDER_TABLE as \"root\"\n" + + " connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" + + " )\n" + + " ) \n" + + " finallyExecutionNodes = \n" + + " (\n" + + " SQL\n" + + " (\n" + + " type = Void\n" + + " resultColumns = []\n" + + " sql = ALTER SESSION UNSET QUERY_TAG;\n" + + " connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" + + " )\n" + + " )\n" + + " )\n" + + " )\n" + + ")\n"; + Assert.assertEquals(snowflakeExpected, snowflakePlan); + } + + + @Test + public void testSemiStructuredMatchMultilevel() + { + String queryFunction = "match::semiStructuredMatchMultilevel__TabularDataSet_1_"; + String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime); + String snowflakeExpected = + " Relational\n" + + " (\n" + + " type = TDS[(Amount, Integer, \"\", \"\")]\n" + + " resultColumns = [(\"Amount\", \"\")]\n" + + " sql = select case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('WalletPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['walletTransactionAmount'] when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CardPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['cardTransactionAmount'] when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountPaid'] else null end when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CashOnDeliveryPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountToBePaid'] else null end as \"Amount\" from ORDER_SCHEMA.ORDER_TABLE as \"root\"\n" + + " connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" + + " )\n"; + String TDSType = " type = TDS[(Amount, Integer, \"\", \"\")]\n"; + Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan); + } + + @Test + public void testSemiStructuredMatchWithMultipleProjectUsingCol() + { + String queryFunction = "match::semiStructuredMatchWithMultipleProjectUsingCol__TabularDataSet_1_"; + String snowflakePlan = this.buildExecutionPlanString(queryFunction, snowflakeMapping, snowflakeRuntime); + String snowflakeExpected = + " Relational\n" + + " (\n" + + " type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n" + + " resultColumns = [(\"Customer Address\", \"\"), (\"Order Price\", \"\")]\n" + + " sql = select case when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('BillingAddress') then \"root\".CUSTOMER['customerAddress']['billAddress']::varchar when \"root\".CUSTOMER['customerAddress']['@type']::varchar in ('ShippingAddress') then \"root\".CUSTOMER['customerAddress']['shipAddress']::varchar else null end as \"Customer Address\", case when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('CashOnDeliveryPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountToBePaid'] when \"root\".CUSTOMER['transactionDetails']['payment']['@type']::varchar in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then \"root\".CUSTOMER['transactionDetails']['payment']['amountPaid'] else null end as \"Order Price\" from ORDER_SCHEMA.ORDER_TABLE as \"root\"\n" + + " connection = RelationalDatabaseConnection(type = \"Snowflake\")\n" + + " )\n"; + String TDSType = " type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n"; + Assert.assertEquals(wrapPreAndFinallyExecutionSqlQuery(TDSType, snowflakeExpected), snowflakePlan); + } + + public String modelResourcePath() + { + return "/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredMatching.pure"; + } +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredMatching.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredMatching.pure new file mode 100644 index 00000000000..765929dba46 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredMatching.pure @@ -0,0 +1,342 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +###Pure +Class match::model::Order +{ + orderId: Integer[1]; + orderName: String[1]; + customer: match::model::Customer[1]; +} + +Class match::model::Customer +{ + customerId: String[1]; + customerAddress: match::model::CustomerAddress[1]; + transactionDetails: match::model::TransactionDetails[1]; +} + +Class match::model::CustomerAddress +{ + name: String[1]; +} + +Class match::model::BillingAddress extends match::model::CustomerAddress +{ + billAddress: String[1]; +} + +Class match::model::ShippingAddress extends match::model::CustomerAddress +{ + shipAddress: String[1]; +} + +Class match::model::TransactionDetails +{ + payment: match::model::Payment[1]; +} + +Class match::model::Payment +{ + paymentId: String[1]; +} + +Class match::model::CashOnDeliveryPayment extends match::model::Payment +{ + amountToBePaid: Integer[1]; +} + +Class match::model::PrepaidPayment extends match::model::Payment +{ + amountPaid: Integer[1]; +} + +Class match::model::WalletPrepaidPayment extends match::model::PrepaidPayment +{ + walletTransactionAmount: Integer[1]; +} + +Class match::model::CardPrepaidPayment extends match::model::PrepaidPayment +{ + cardTransactionAmount: Integer[1]; +} + +###Relational +Database match::store::SnowflakeDB +( + Schema ORDER_SCHEMA + ( + Table ORDER_TABLE + ( + ORDERID INTEGER PRIMARY KEY, + ORDERNAME VARCHAR(100), + CUSTOMER SEMISTRUCTURED + ) + ) +) + +Database match::store::MemSQLDB +( + Schema ORDER_SCHEMA + ( + Table ORDER_TABLE + ( + ORDERID INTEGER PRIMARY KEY, + ORDERNAME VARCHAR(100), + CUSTOMER JSON + ) + ) +) + +Database match::store::H2DB +( + Schema ORDER_SCHEMA + ( + Table ORDER_TABLE + ( + ORDERID INTEGER PRIMARY KEY, + ORDERNAME VARCHAR(100), + CUSTOMER VARCHAR(1000) + ) + ) +) + +###ExternalFormat +Binding match::store::OrderBinding +{ + contentType: 'application/json'; + modelIncludes: [ + match::model::Order, + match::model::Customer, + match::model::CustomerAddress, + match::model::BillingAddress, + match::model::ShippingAddress, + match::model::TransactionDetails, + match::model::Payment, + match::model::CashOnDeliveryPayment, + match::model::PrepaidPayment, + match::model::WalletPrepaidPayment, + match::model::CardPrepaidPayment + ]; +} + +###Mapping +Mapping match::mapping::SnowflakeMapping +( + match::model::Order: Relational + { + ~primaryKey + ( + [match::store::SnowflakeDB]ORDER_SCHEMA.ORDER_TABLE.ORDERID + ) + ~mainTable [match::store::SnowflakeDB]ORDER_SCHEMA.ORDER_TABLE + orderId: [match::store::SnowflakeDB]ORDER_SCHEMA.ORDER_TABLE.ORDERID, + orderName: [match::store::SnowflakeDB]ORDER_SCHEMA.ORDER_TABLE.ORDERNAME, + customer: Binding match::store::OrderBinding : [match::store::SnowflakeDB]ORDER_SCHEMA.ORDER_TABLE.CUSTOMER + } +) + +Mapping match::mapping::MemSQLMapping +( + match::model::Order: Relational + { + ~primaryKey + ( + [match::store::MemSQLDB]ORDER_SCHEMA.ORDER_TABLE.ORDERID + ) + ~mainTable [match::store::MemSQLDB]ORDER_SCHEMA.ORDER_TABLE + orderId: [match::store::MemSQLDB]ORDER_SCHEMA.ORDER_TABLE.ORDERID, + orderName: [match::store::MemSQLDB]ORDER_SCHEMA.ORDER_TABLE.ORDERNAME, + customer: Binding match::store::OrderBinding : [match::store::MemSQLDB]ORDER_SCHEMA.ORDER_TABLE.CUSTOMER + } +) + +Mapping match::mapping::H2Mapping +( + match::model::Order: Relational + { + ~primaryKey + ( + [match::store::H2DB]ORDER_SCHEMA.ORDER_TABLE.ORDERID + ) + ~mainTable [match::store::H2DB]ORDER_SCHEMA.ORDER_TABLE + orderId: [match::store::H2DB]ORDER_SCHEMA.ORDER_TABLE.ORDERID, + orderName: [match::store::H2DB]ORDER_SCHEMA.ORDER_TABLE.ORDERNAME, + customer: Binding match::store::OrderBinding : [match::store::H2DB]ORDER_SCHEMA.ORDER_TABLE.CUSTOMER + } +) + +###Runtime +Runtime match::runtime::SnowflakeRuntime +{ + mappings : + [ + match::mapping::SnowflakeMapping + ]; + connections : + [ + match::store::SnowflakeDB : + [ + connection_1 : #{ + RelationalDatabaseConnection { + store: match::store::SnowflakeDB; + type: Snowflake; + specification: Snowflake + { + name: 'dbName'; + account: 'account'; + warehouse: 'warehouse'; + region: 'region'; + }; + auth: Test; + } + }# + ] + ]; +} + +###Pure +function match::semiStructuredMatchComplexProperty(): TabularDataSet[1] +{ + match::model::Order.all()->project( + [ + x | $x.customer.customerAddress->match( + [ + s: match::model::BillingAddress[1] | $s.billAddress, + s: match::model::ShippingAddress[1] | $s.shipAddress, + s: Any[1] | 'Default Address' + ] + ) + ], + [ + 'Customer Address' + ] + ); +} + +function match::semiStructuredMatchWithMultipleProject(): TabularDataSet[1] +{ + match::model::Order.all()->project( + [ + x | $x.customer.customerAddress->match( + [ + s: match::model::BillingAddress[1] | $s.billAddress, + s: match::model::ShippingAddress[1] | $s.shipAddress + ] + ), + x | $x.customer.transactionDetails.payment->match( + [ + s:match::model::CashOnDeliveryPayment[1] | $s.amountToBePaid, + s:match::model::PrepaidPayment[1] | $s.amountPaid + ] + ) + ], + [ + 'Customer Address', + 'Order Price' + ] + ); +} + +function match::semiStructuredMatchWithComplexFilter(): TabularDataSet[1] +{ + match::model::Order.all() + ->filter( + x | $x.customer.transactionDetails.payment->match( + [ + s:match::model::CashOnDeliveryPayment[1] | $s.amountToBePaid, + s:match::model::PrepaidPayment[1] | $s.amountPaid + ] + )<200 + ) + ->project( + [ + x | $x.customer.customerAddress->match( + [ + s: match::model::BillingAddress[1] | $s.billAddress, + s: match::model::ShippingAddress[1] | $s.shipAddress + ] + ) + ], + [ + 'Customer Address' + ] + ); +} + +function match::semiStructuredMatchWithVariableAccess(): TabularDataSet[1] +{ + let maxAmount = 200; + match::model::Order.all()->project( + [ + x | $x.customer.transactionDetails.payment->match( + [ + s:match::model::CashOnDeliveryPayment[1] | + if($s.amountToBePaid < $maxAmount, + |true, + |false + ), + s:match::model::PrepaidPayment[1] | + if($s.amountPaid < $maxAmount, + |true, + |false + ) + ] + ) + ], + [ + 'Max Amount Flag' + ] + ); + +} + +function match::semiStructuredMatchMultilevel(): TabularDataSet[1] +{ + match::model::Order.all()->project( + [ + x | $x.customer.transactionDetails.payment->match( + [ + s: match::model::PrepaidPayment[1] | $s->match( + [ + k:match::model::WalletPrepaidPayment[1] | $k.walletTransactionAmount , + k:match::model::CardPrepaidPayment[1] | $k.cardTransactionAmount, + k:match::model::PrepaidPayment[1] | $k.amountPaid + ] + ), + s: match::model::CashOnDeliveryPayment[1] | $s.amountToBePaid + ] + ) + ], + [ + 'Amount' + ] + ); +} + +function match::semiStructuredMatchWithMultipleProjectUsingCol(): TabularDataSet[1] +{ + match::model::Order.all()->project([ + col(x | $x.customer.customerAddress->match( + [ + s: match::model::BillingAddress[1] | $s.billAddress, + s: match::model::ShippingAddress[1] | $s.shipAddress + ]),'Customer Address'), + col(x | $x.customer.transactionDetails.payment->match( + [ + s:match::model::CashOnDeliveryPayment[1] | $s.amountToBePaid, + s:match::model::PrepaidPayment[1] | $s.amountPaid + ]),'Order Price') + ]); +} \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSemiStructuredMatching.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSemiStructuredMatching.java new file mode 100644 index 00000000000..94eaeb9fe4d --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/TestSemiStructuredMatching.java @@ -0,0 +1,219 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.plan.execution.stores.relational.test.semiStructured; + +import org.junit.Assert; +import org.junit.Test; + +public class TestSemiStructuredMatching extends AbstractTestSemiStructured +{ + private static final String memSQLMapping = "match::mapping::MemSQLMapping"; + private static final String memSQLRuntime = "match::runtime::MemSQLRuntime"; + + private static final String h2Mapping = "match::mapping::H2Mapping"; + private static final String h2Runtime = "match::runtime::H2Runtime"; + + @Test + public void testSemiStructuredMatchComplexProperty() + { + String queryFunction = "match::semiStructuredMatchComplexProperty__TabularDataSet_1_"; + String memSQLPlan = this.buildExecutionPlanString(queryFunction, memSQLMapping, memSQLRuntime); + String memSQLExpected = + "Relational\n" + + "(\n" + + " type = TDS[(Customer Address, String, \"\", \"\")]\n" + + " resultColumns = [(\"Customer Address\", \"\")]\n" + + " sql = select case when `root`.CUSTOMER::customerAddress::$@type in ('BillingAddress') then `root`.CUSTOMER::customerAddress::$billAddress when `root`.CUSTOMER::customerAddress::$@type in ('ShippingAddress') then `root`.CUSTOMER::customerAddress::$shipAddress else 'Default Address' end as `Customer Address` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" + + " connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" + + ")\n"; + Assert.assertEquals(memSQLExpected, memSQLPlan); + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("B1\n" + + "B2\n" + + "B3\n" + + "Default Address\n" + + "S1\n" + + "S2\n" + + "S3\n" + + "S4\n", h2Result.replace("\r\n", "\n")); + + Assert.assertEquals("[ORDER_TABLE.CUSTOMER ]", this.scanColumns(queryFunction, h2Mapping)); + } + + @Test + public void testSemiStructuredMatchWithMultipleProject() + { + String queryFunction = "match::semiStructuredMatchWithMultipleProject__TabularDataSet_1_"; + String memSQLPlan = this.buildExecutionPlanString(queryFunction, memSQLMapping, memSQLRuntime); + String memSQLExpected = + "Relational\n" + + "(\n" + + " type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n" + + " resultColumns = [(\"Customer Address\", \"\"), (\"Order Price\", \"\")]\n" + + " sql = select case when `root`.CUSTOMER::customerAddress::$@type in ('BillingAddress') then `root`.CUSTOMER::customerAddress::$billAddress when `root`.CUSTOMER::customerAddress::$@type in ('ShippingAddress') then `root`.CUSTOMER::customerAddress::$shipAddress else null end as `Customer Address`, case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountToBePaid when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountPaid else null end as `Order Price` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" + + " connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" + + ")\n"; + Assert.assertEquals(memSQLExpected, memSQLPlan); + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("B1,200\n" + + "B2,180\n" + + "B3,290\n" + + ",150\n" + + "S1,185\n" + + "S2,120\n" + + "S3,180\n" + + "S4,160\n", h2Result.replace("\r\n", "\n")); + + Assert.assertEquals("[ORDER_TABLE.CUSTOMER ]", this.scanColumns(queryFunction, h2Mapping)); + } + + @Test + public void testSemiStructuredMatchWithComplexFilter() + { + String queryFunction = "match::semiStructuredMatchWithComplexFilter__TabularDataSet_1_"; + String memSQLPlan = this.buildExecutionPlanString(queryFunction, memSQLMapping, memSQLRuntime); + String memSQLExpected = + "Relational\n" + + "(\n" + + " type = TDS[(Customer Address, String, \"\", \"\")]\n" + + " resultColumns = [(\"Customer Address\", \"\")]\n" + + " sql = select case when `root`.CUSTOMER::customerAddress::$@type in ('BillingAddress') then `root`.CUSTOMER::customerAddress::$billAddress when `root`.CUSTOMER::customerAddress::$@type in ('ShippingAddress') then `root`.CUSTOMER::customerAddress::$shipAddress else null end as `Customer Address` from ORDER_SCHEMA.ORDER_TABLE as `root` where case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountToBePaid when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountPaid else null end < 200\n" + + " connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" + + ")\n"; + Assert.assertEquals(memSQLExpected, memSQLPlan); + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("B2\n" + + "\n" + + "S1\n" + + "S2\n" + + "S3\n" + + "S4\n", h2Result.replace("\r\n", "\n")); + + Assert.assertEquals("[ORDER_TABLE.CUSTOMER ]", this.scanColumns(queryFunction, h2Mapping)); + } + + @Test + public void testSemiStructuredMatchWithVariableAccess() + { + String queryFunction = "match::semiStructuredMatchWithVariableAccess__TabularDataSet_1_"; + String memSQLPlan = this.buildExecutionPlanString(queryFunction, memSQLMapping, memSQLRuntime); + String memSQLExpected = + "Sequence\n" + + "(\n" + + " type = TDS[(Max Amount Flag, Boolean, \"\", \"\")]\n" + + " (\n" + + " Allocation\n" + + " (\n" + + " type = Integer\n" + + " resultSizeRange = 1\n" + + " name = maxAmount\n" + + " value = \n" + + " (\n" + + " Constant\n" + + " (\n" + + " type = Integer\n" + + " resultSizeRange = 1\n" + + " values=[200]\n" + + " )\n" + + " )\n" + + " )\n" + + " Relational\n" + + " (\n" + + " type = TDS[(Max Amount Flag, Boolean, \"\", \"\")]\n" + + " resultColumns = [(\"Max Amount Flag\", \"\")]\n" + + " sql = select case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then case when `root`.CUSTOMER::transactionDetails::payment::%amountToBePaid < ${maxAmount} then 'true' else 'false' end when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then case when `root`.CUSTOMER::transactionDetails::payment::%amountPaid < ${maxAmount} then 'true' else 'false' end else null end as `Max Amount Flag` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" + + " connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" + + " )\n" + + " )\n" + + ")\n"; + Assert.assertEquals(memSQLExpected, memSQLPlan); + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("false\n" + + "true\n" + + "false\n" + + "true\n" + + "true\n" + + "true\n" + + "true\n" + + "true\n", h2Result.replace("\r\n", "\n")); + + } + + + @Test + public void testSemiStructuredMatchMultilevel() + { + String queryFunction = "match::semiStructuredMatchMultilevel__TabularDataSet_1_"; + String memSQLPlan = this.buildExecutionPlanString(queryFunction, memSQLMapping, memSQLRuntime); + String memSQLExpected = + "Relational\n" + + "(\n" + + " type = TDS[(Amount, Integer, \"\", \"\")]\n" + + " resultColumns = [(\"Amount\", \"\")]\n" + + " sql = select case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('WalletPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%walletTransactionAmount when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%cardTransactionAmount when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountPaid else null end when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountToBePaid else null end as `Amount` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" + + " connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" + + ")\n"; + Assert.assertEquals(memSQLExpected, memSQLPlan); + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("200\n" + + "180\n" + + "290\n" + + "150\n" + + "185\n" + + "120\n" + + "200\n" + + "190\n", h2Result.replace("\r\n", "\n")); + + Assert.assertEquals("[ORDER_TABLE.CUSTOMER ]", this.scanColumns(queryFunction, h2Mapping)); + } + + @Test + public void testSemiStructuredMatchWithMultipleProjectUsingCol() + { + String queryFunction = "match::semiStructuredMatchWithMultipleProjectUsingCol__TabularDataSet_1_"; + String memSQLPlan = this.buildExecutionPlanString(queryFunction, memSQLMapping, memSQLRuntime); + String memSQLExpected = + "Relational\n" + + "(\n" + + " type = TDS[(Customer Address, String, \"\", \"\"), (Order Price, Integer, \"\", \"\")]\n" + + " resultColumns = [(\"Customer Address\", \"\"), (\"Order Price\", \"\")]\n" + + " sql = select case when `root`.CUSTOMER::customerAddress::$@type in ('BillingAddress') then `root`.CUSTOMER::customerAddress::$billAddress when `root`.CUSTOMER::customerAddress::$@type in ('ShippingAddress') then `root`.CUSTOMER::customerAddress::$shipAddress else null end as `Customer Address`, case when `root`.CUSTOMER::transactionDetails::payment::$@type in ('CashOnDeliveryPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountToBePaid when `root`.CUSTOMER::transactionDetails::payment::$@type in ('PrepaidPayment', 'WalletPrepaidPayment', 'CardPrepaidPayment') then `root`.CUSTOMER::transactionDetails::payment::%amountPaid else null end as `Order Price` from ORDER_SCHEMA.ORDER_TABLE as `root`\n" + + " connection = RelationalDatabaseConnection(type = \"MemSQL\")\n" + + ")\n"; + Assert.assertEquals(memSQLExpected, memSQLPlan); + + String h2Result = this.executeFunction(queryFunction, h2Mapping, h2Runtime); + Assert.assertEquals("B1,200\n" + + "B2,180\n" + + "B3,290\n" + + ",150\n" + + "S1,185\n" + + "S2,120\n" + + "S3,180\n" + + "S4,160\n", h2Result.replace("\r\n", "\n")); + + Assert.assertEquals("[ORDER_TABLE.CUSTOMER ]", this.scanColumns(queryFunction, h2Mapping)); + } + + public String modelResourcePath() + { + return "/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredMatching.pure"; + } +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredMatching.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredMatching.pure new file mode 100644 index 00000000000..b0ebf3d9f20 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/test/resources/org/finos/legend/engine/plan/execution/stores/relational/test/semiStructured/semiStructuredMatching.pure @@ -0,0 +1,348 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +###Pure +Class match::model::Order +{ + orderId: Integer[1]; + orderName: String[1]; + customer: match::model::Customer[1]; +} + +Class match::model::Customer +{ + customerId: String[1]; + customerAddress: match::model::CustomerAddress[1]; + transactionDetails: match::model::TransactionDetails[1]; +} + +Class match::model::CustomerAddress +{ + name: String[1]; +} + +Class match::model::BillingAddress extends match::model::CustomerAddress +{ + billAddress: String[1]; +} + +Class match::model::ShippingAddress extends match::model::CustomerAddress +{ + shipAddress: String[1]; +} + +Class match::model::TransactionDetails +{ + payment: match::model::Payment[1]; +} + +Class match::model::Payment +{ + paymentId: String[1]; +} + +Class match::model::CashOnDeliveryPayment extends match::model::Payment +{ + amountToBePaid: Integer[1]; +} + +Class match::model::PrepaidPayment extends match::model::Payment +{ + amountPaid: Integer[1]; +} + +Class match::model::WalletPrepaidPayment extends match::model::PrepaidPayment +{ + walletTransactionAmount: Integer[1]; +} + +Class match::model::CardPrepaidPayment extends match::model::PrepaidPayment +{ + cardTransactionAmount: Integer[1]; +} + +###Relational +Database match::store::MemSQLDB +( + Schema ORDER_SCHEMA + ( + Table ORDER_TABLE + ( + ORDERID INTEGER PRIMARY KEY, + ORDERNAME VARCHAR(100), + CUSTOMER JSON + ) + ) +) + +Database match::store::H2DB +( + Schema ORDER_SCHEMA + ( + Table ORDER_TABLE + ( + ORDERID INTEGER PRIMARY KEY, + ORDERNAME VARCHAR(100), + CUSTOMER VARCHAR(1000) + ) + ) +) + +###ExternalFormat +Binding match::store::OrderBinding +{ + contentType: 'application/json'; + modelIncludes: [ + match::model::Order, + match::model::Customer, + match::model::CustomerAddress, + match::model::BillingAddress, + match::model::ShippingAddress, + match::model::TransactionDetails, + match::model::Payment, + match::model::CashOnDeliveryPayment, + match::model::PrepaidPayment, + match::model::WalletPrepaidPayment, + match::model::CardPrepaidPayment + ]; +} + +###Mapping +Mapping match::mapping::MemSQLMapping +( + match::model::Order: Relational + { + ~primaryKey + ( + [match::store::MemSQLDB]ORDER_SCHEMA.ORDER_TABLE.ORDERID + ) + ~mainTable [match::store::MemSQLDB]ORDER_SCHEMA.ORDER_TABLE + orderId: [match::store::MemSQLDB]ORDER_SCHEMA.ORDER_TABLE.ORDERID, + orderName: [match::store::MemSQLDB]ORDER_SCHEMA.ORDER_TABLE.ORDERNAME, + customer: Binding match::store::OrderBinding : [match::store::MemSQLDB]ORDER_SCHEMA.ORDER_TABLE.CUSTOMER + } +) + +Mapping match::mapping::H2Mapping +( + match::model::Order: Relational + { + ~primaryKey + ( + [match::store::H2DB]ORDER_SCHEMA.ORDER_TABLE.ORDERID + ) + ~mainTable [match::store::H2DB]ORDER_SCHEMA.ORDER_TABLE + orderId: [match::store::H2DB]ORDER_SCHEMA.ORDER_TABLE.ORDERID, + orderName: [match::store::H2DB]ORDER_SCHEMA.ORDER_TABLE.ORDERNAME, + customer: Binding match::store::OrderBinding : [match::store::H2DB]ORDER_SCHEMA.ORDER_TABLE.CUSTOMER + } +) + + +###Runtime +Runtime match::runtime::MemSQLRuntime +{ + mappings : + [ + match::mapping::MemSQLMapping + ]; + connections : + [ + match::store::MemSQLDB : + [ + connection_1 : #{ + RelationalDatabaseConnection { + store: match::store::MemSQLDB; + type: MemSQL; + specification: LocalH2{}; + auth: Test; + } + }# + ] + ]; +} + +Runtime match::runtime::H2Runtime +{ + mappings : + [ + match::mapping::H2Mapping + ]; + connections : + [ + match::store::H2DB : + [ + connection_1 : #{ + RelationalDatabaseConnection { + store: match::store::H2DB; + type: H2; + specification: LocalH2{ + testDataSetupSqls: [ + 'DROP SCHEMA IF EXISTS ORDER_SCHEMA CASCADE;', + 'CREATE SCHEMA ORDER_SCHEMA;', + 'CREATE TABLE ORDER_SCHEMA.ORDER_TABLE(ORDERID INT PRIMARY KEY, ORDERNAME VARCHAR(100), CUSTOMER VARCHAR(1000));', + 'INSERT INTO ORDER_SCHEMA.ORDER_TABLE(ORDERID,ORDERNAME,CUSTOMER) VALUES (1,\'Order1\',\'{"customerId": "Customer1", "customerAddress": {"@type":"BillingAddress","name": "A1","billAddress":"B1"},"transactionDetails": {"payment":{"@type":"CashOnDeliveryPayment","paymentId": "P1","amountToBePaid": 200}}}\');', + 'INSERT INTO ORDER_SCHEMA.ORDER_TABLE(ORDERID,ORDERNAME,CUSTOMER) VALUES (2,\'Order2\',\'{"customerId": "Customer2", "customerAddress": {"@type":"BillingAddress","name": "A2","billAddress":"B2"},"transactionDetails": {"payment":{"@type":"CashOnDeliveryPayment","paymentId": "P2","amountToBePaid": 180}}}\');', + 'INSERT INTO ORDER_SCHEMA.ORDER_TABLE(ORDERID,ORDERNAME,CUSTOMER) VALUES (3,\'Order3\',\'{"customerId": "Customer3", "customerAddress": {"@type":"BillingAddress","name": "A3","billAddress":"B3"},"transactionDetails": {"payment":{"@type":"CashOnDeliveryPayment","paymentId": "P3","amountToBePaid": 290}}}\');', + 'INSERT INTO ORDER_SCHEMA.ORDER_TABLE(ORDERID,ORDERNAME,CUSTOMER) VALUES (4,\'Order4\',\'{"customerId": "Customer4", "customerAddress": {"name": "A4"},"transactionDetails": {"payment":{"@type":"PrepaidPayment","paymentId": "P4","amountPaid": 150}}}\');', + 'INSERT INTO ORDER_SCHEMA.ORDER_TABLE(ORDERID,ORDERNAME,CUSTOMER) VALUES (6,\'Order6\',\'{"customerId": "Customer6", "customerAddress": {"@type":"ShippingAddress","name": "A6","shipAddress":"S2"},"transactionDetails": {"payment":{"@type":"PrepaidPayment","paymentId": "P6","amountPaid": 120}}}\')', + 'INSERT INTO ORDER_SCHEMA.ORDER_TABLE(ORDERID,ORDERNAME,CUSTOMER) VALUES (5,\'Order5\',\'{"customerId": "Customer5", "customerAddress": {"@type":"ShippingAddress","name": "A5","shipAddress":"S1"},"transactionDetails": {"payment":{"@type":"PrepaidPayment","paymentId": "P5","amountPaid": 185}}}\')', + 'INSERT INTO ORDER_SCHEMA.ORDER_TABLE(ORDERID,ORDERNAME,CUSTOMER) VALUES (7,\'Order7\',\'{"customerId": "Customer7", "customerAddress": {"@type":"ShippingAddress","name": "A7","shipAddress":"S3"},"transactionDetails": {"payment":{"@type":"WalletPrepaidPayment","paymentId": "P7","amountPaid": 180,"walletTransactionAmount":200}}}\')', + 'INSERT INTO ORDER_SCHEMA.ORDER_TABLE(ORDERID,ORDERNAME,CUSTOMER) VALUES (8,\'Order8\',\'{"customerId": "Customer8", "customerAddress": {"@type":"ShippingAddress","name": "A8","shipAddress":"S4"},"transactionDetails": {"payment":{"@type":"CardPrepaidPayment","paymentId": "P8","amountPaid": 160, "cardTransactionAmount":190}}}\')' + + ]; + }; + auth: Test; + } + }# + ] + ]; +} + + + +###Pure +function match::semiStructuredMatchComplexProperty(): TabularDataSet[1] +{ + match::model::Order.all()->project( + [ + x | $x.customer.customerAddress->match( + [ + s: match::model::BillingAddress[1] | $s.billAddress, + s: match::model::ShippingAddress[1] | $s.shipAddress, + s: Any[1] | 'Default Address' + ] + ) + ], + [ + 'Customer Address' + ] + ); +} + +function match::semiStructuredMatchWithMultipleProject(): TabularDataSet[1] +{ + match::model::Order.all()->project( + [ + x | $x.customer.customerAddress->match( + [ + s: match::model::BillingAddress[1] | $s.billAddress, + s: match::model::ShippingAddress[1] | $s.shipAddress + ] + ), + x | $x.customer.transactionDetails.payment->match( + [ + s:match::model::CashOnDeliveryPayment[1] | $s.amountToBePaid, + s:match::model::PrepaidPayment[1] | $s.amountPaid + ] + ) + ], + [ + 'Customer Address', + 'Order Price' + ] + ); +} + +function match::semiStructuredMatchWithComplexFilter(): TabularDataSet[1] +{ + match::model::Order.all() + ->filter( + x | $x.customer.transactionDetails.payment->match( + [ + s:match::model::CashOnDeliveryPayment[1] | $s.amountToBePaid, + s:match::model::PrepaidPayment[1] | $s.amountPaid + ] + )<200 + ) + ->project( + [ + x | $x.customer.customerAddress->match( + [ + s: match::model::BillingAddress[1] | $s.billAddress, + s: match::model::ShippingAddress[1] | $s.shipAddress + ] + ) + ], + [ + 'Customer Address' + ] + ); +} + +function match::semiStructuredMatchWithVariableAccess(): TabularDataSet[1] +{ + let maxAmount = 200; + match::model::Order.all()->project( + [ + x | $x.customer.transactionDetails.payment->match( + [ + s:match::model::CashOnDeliveryPayment[1] | + if($s.amountToBePaid < $maxAmount, + |true, + |false + ), + s:match::model::PrepaidPayment[1] | + if($s.amountPaid < $maxAmount, + |true, + |false + ) + ] + ) + ], + [ + 'Max Amount Flag' + ] + ); + +} + +function match::semiStructuredMatchMultilevel(): TabularDataSet[1] +{ + match::model::Order.all()->project( + [ + x | $x.customer.transactionDetails.payment->match( + [ + s: match::model::PrepaidPayment[1] | $s->match( + [ + k:match::model::WalletPrepaidPayment[1] | $k.walletTransactionAmount , + k:match::model::CardPrepaidPayment[1] | $k.cardTransactionAmount, + k:match::model::PrepaidPayment[1] | $k.amountPaid + ] + ), + s: match::model::CashOnDeliveryPayment[1] | $s.amountToBePaid + ] + ) + ], + [ + 'Amount' + ] + ); +} + +function match::semiStructuredMatchWithMultipleProjectUsingCol(): TabularDataSet[1] +{ + match::model::Order.all()->project([ + col(x | $x.customer.customerAddress->match( + [ + s: match::model::BillingAddress[1] | $s.billAddress, + s: match::model::ShippingAddress[1] | $s.shipAddress + ]),'Customer Address'), + col(x | $x.customer.transactionDetails.payment->match( + [ + s:match::model::CashOnDeliveryPayment[1] | $s.amountToBePaid, + s:match::model::PrepaidPayment[1] | $s.amountPaid + ]),'Order Price') + ]); +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure index 97e867c21af..a7e689058c7 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure @@ -2319,6 +2319,67 @@ function <> meta::relational::functions::pureToSqlQuery::process ^$processedLeftSide(element = ^$collectionSWC(select = $newSelect)); } + +function <> meta::relational::functions::pureToSqlQuery::canProcessMatch(f:FunctionExpression[1]):Boolean[1] +{ + // Can process 'match' in the context of semi structured mappings (which are not routed). + let params = $f.parametersValues->evaluateAndDeactivate(); + ($params->size() == 2) && (!$params->at(0)->instanceOf(RoutedValueSpecification) || $params->at(0)->instanceOf(NoSetRoutedValueSpecification)); +} + +function <> meta::relational::functions::pureToSqlQuery::processMatch(expression:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1] +{ + assert($expression.genericType.rawType->toOne()->instanceOf(PrimitiveType),'Match does not support Non-Primitive return type..! Current return type : '+ $expression.genericType.rawType->toOne()->toString()); + let leftSide = $expression.parametersValues->at(0); + let byPassedLeftSide = $leftSide->deepByPassRouterInfo(); + + assert($byPassedLeftSide->cast(@ValueSpecification).multiplicity->isToOne(), 'Match only supports operands with multiplicity [1]..! Current operand : '+ $byPassedLeftSide->cast(@ValueSpecification)->meta::pure::router::printer::asString()); + let leftSideOp = processValueSpecificationReturnPropertyMapping($leftSide, $currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions)->toOne(); + let processedLeftSide = $leftSideOp.element->cast(@SelectWithCursor); + let leftSelect = $processedLeftSide.select; + + let checkCol = if($state.inFilter,|$leftSelect.filteringOperation, |$leftSelect.columns)->toOne(); + //Currently hardcoding '@type' as a property. + let checkColType = ^SemiStructuredPropertyAccess(operand = $checkCol->cast(@SemiStructuredPropertyAccess),property = ^Literal(value = '@type'), returnType = String); + let propMapping = $leftSideOp.currentPropertyMapping; + + let functions = $expression->instanceValuesAtParameter(1, $vars, $state.inScopeVars)->cast(@meta::pure::metamodel::function::Function); + let processedFuncs = $functions->map(f|$f->processMatchFunctions($processedLeftSide, $vars, $state, $nodeId, $aggFromMap, $context->shift(), $extensions, $joinType, $propMapping)); + + let matchConclusions = $processedFuncs->map(x | let select = $x.element->cast(@SelectWithCursor).select; + if($state.inFilter,|$select.filteringOperation, |$select.columns)->toOne(); + ); + let functionCount = $functions->size(); + let allFunctionsSubtypes = $functions->size()->range()->map(ind | let functionType = $functions->at($ind)->meta::pure::functions::meta::functionType().parameters->at(0).genericType.rawType->toOne(); + assertFalse($functionType->instanceOf(PrimitiveType), 'Match does not support checking on primitive data type..! Currently checking on : ' + $functionType->toString()); + if($functionType->isAnyClass(), + | assert($ind == $functionCount-1, 'Any should be used as a default type at the end of match'); + [];, + | + list($functionType->cast(@Class)->meta::pure::functions::meta::allSpecializations()) + ); + ); + + let matchConditions = $allFunctionsSubtypes->size()->range()->map(index | ^DynaFunction(name = 'in', parameters = [$checkColType,^LiteralList(values = $allFunctionsSubtypes->at($index).values.name->map(x | ^meta::relational::metamodel::Literal(value =$x)))])); + let matchConditionsWithConclusions = $allFunctionsSubtypes->size()->range()->map(index | $matchConditions->at($index)->concatenate($matchConclusions->at($index))); + + let caseParams = if($matchConditions->size() == $functionCount - 1, + |$matchConditionsWithConclusions->concatenate($matchConclusions->at($functionCount - 1)), + |$matchConditionsWithConclusions->concatenate(^meta::relational::metamodel::Literal(value=^SQLNull())) + ); + + let newSelect = $state.inFilter->if(|^$leftSelect(filteringOperation = ^DynaFunction(name = 'case', parameters = $caseParams)),|^$leftSelect(columns = ^DynaFunction(name = 'case', parameters = $caseParams))); + ^$processedLeftSide(select =$newSelect); +} + +function <> meta::relational::functions::pureToSqlQuery::processMatchFunctions(f:meta::pure::metamodel::function::Function[1], query:SelectWithCursor[1], vars:Map[1], state:State[1], nodeId:String[1], aggFromMap:List[1], context:DebugContext[1], extensions:Extension[*], joinType:JoinType[1], propMapping : PropertyMapping[*]):OperationWithParentPropertyMapping[1] +{ + assert($f->functionReturnMultiplicity()->isToOne(),'Match only supports lambdas with multiplicity [1]..! Current lambda : ' + $f->deepByPassRouterInfo()->cast(@meta::pure::metamodel::function::Function)->meta::pure::router::printer::asString()); + let expression = $f->cast(@meta::pure::metamodel::function::FunctionDefinition).expressionSequence; + assertSize($expression, 1,'Lambdas with more than one functionExpressions are not supported yet in Match..! The lambda \'' + $f->deepByPassRouterInfo()->cast(@meta::pure::metamodel::function::Function)->meta::pure::router::printer::asString() + '\' has ' + $expression->size()->toString() + ' expressions.'); + processValueSpecificationReturnPropertyMapping($expression->toOne(),$propMapping, $query, $vars, $state, $joinType, $nodeId, $aggFromMap, $context->shift(), $extensions)->toOne(); +} + function meta::relational::functions::pureToSqlQuery::processNot(f:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1] { processUnary($f, $currentPropertyMapping, $operation, {a|^DynaFunction(name = 'not', parameters = $a)}, $vars, $state, $nodeId, $aggFromMap, $context, $extensions)->cast(@SelectWithCursor)->moveExtraFilterToFilter($extensions) @@ -7463,6 +7524,13 @@ function meta::relational::functions::pureToSqlQuery::getContextBasedSupportedFu meta::relational::functions::pureToSqlQuery::canProcessAt_FunctionExpression_1__Boolean_1_, meta::relational::functions::pureToSqlQuery::processAt_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_ ) + ), + ^Pair, PairBoolean[1]}>, meta::pure::metamodel::function::Function<{FunctionExpression[1], PropertyMapping[0..1], SelectWithCursor[1], Map[1], State[1], JoinType[1], String[1], List[1], DebugContext[1], Extension[*]->RelationalOperationElement[1]}>>>( + first = meta::pure::functions::lang::match_Any_MANY__Function_$1_MANY$__T_m_, + second = pair( + meta::relational::functions::pureToSqlQuery::canProcessMatch_FunctionExpression_1__Boolean_1_, + meta::relational::functions::pureToSqlQuery::processMatch_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_ + ) ) ]) }