From 7fba51ab7a292fe30c5fe3684acfc4719f925d44 Mon Sep 17 00:00:00 2001 From: tobe Date: Fri, 29 Dec 2023 18:47:19 +0800 Subject: [PATCH 1/4] Refactor test feature service form --- .../featureservice/FeatureServicesData.vue | 12 +- .../featureservice/OnlineQueryModePage.vue | 353 ++++++++++++++++++ .../featureservice/RequestFeatureService.vue | 265 +++++++++++++ .../featureservice/RequestModePage.vue | 250 +++++++++++++ .../form/TestFeatureServiceForm.vue | 9 +- frontend/src/locales/en.json | 3 +- frontend/src/locales/zh.json | 5 +- frontend/src/router/index.js | 7 + 8 files changed, 886 insertions(+), 18 deletions(-) create mode 100644 frontend/src/components/featureservice/OnlineQueryModePage.vue create mode 100644 frontend/src/components/featureservice/RequestFeatureService.vue create mode 100644 frontend/src/components/featureservice/RequestModePage.vue diff --git a/frontend/src/components/featureservice/FeatureServicesData.vue b/frontend/src/components/featureservice/FeatureServicesData.vue index 98e94ae..5142a02 100644 --- a/frontend/src/components/featureservice/FeatureServicesData.vue +++ b/frontend/src/components/featureservice/FeatureServicesData.vue @@ -15,10 +15,6 @@ - - - -

@@ -35,7 +31,7 @@ {{ record.version }} @@ -45,13 +41,11 @@ \ No newline at end of file diff --git a/frontend/src/components/featureservice/RequestFeatureService.vue b/frontend/src/components/featureservice/RequestFeatureService.vue new file mode 100644 index 0000000..3546f3a --- /dev/null +++ b/frontend/src/components/featureservice/RequestFeatureService.vue @@ -0,0 +1,265 @@ + + + \ No newline at end of file diff --git a/frontend/src/components/featureservice/RequestModePage.vue b/frontend/src/components/featureservice/RequestModePage.vue new file mode 100644 index 0000000..f10ef8b --- /dev/null +++ b/frontend/src/components/featureservice/RequestModePage.vue @@ -0,0 +1,250 @@ + + + \ No newline at end of file diff --git a/frontend/src/components/form/TestFeatureServiceForm.vue b/frontend/src/components/form/TestFeatureServiceForm.vue index 09575c1..6555f92 100644 --- a/frontend/src/components/form/TestFeatureServiceForm.vue +++ b/frontend/src/components/form/TestFeatureServiceForm.vue @@ -4,7 +4,8 @@
-
{{ $t("Text of introduce test feature service") }} {{$t('OpenMLDB documents')}}
+ +
用户可以使用“请求模式”或“在线查询模式”来访问特征服务,请求模式下用户需要输入整行主表数据通过 LastJoin 等方法提取副表特征;在线查询模式则不需要传入数据,直接查询在线表的特征数据,并且可以通过主键进行数据过滤。

@@ -28,7 +29,6 @@ - @@ -43,7 +43,8 @@   {{ $t('Switch Json Data') }}

- + +
@@ -61,7 +62,7 @@
- +
Date: Wed, 3 Jan 2024 18:43:03 +0800 Subject: [PATCH 2/4] Support online query mode --- .../featureservice/FeatureServiceDetail.vue | 2 +- .../FeatureServiceVersionDetail.vue | 4 +- .../featureservice/OnlineQueryModePage.vue | 414 +++++++----------- .../featureservice/RequestFeatureService.vue | 19 - .../featureservice/RequestModePage.vue | 21 +- frontend/src/locales/en.json | 4 +- frontend/src/locales/zh.json | 4 +- .../controller/FeatureServiceController.java | 22 + .../service/FeatureServiceService.java | 110 ++++- .../featureplatform/service/TableService.java | 5 +- .../utils/OpenmldbSqlUtil.java | 30 ++ .../utils/OpenmldbTableUtil.java | 23 + 12 files changed, 367 insertions(+), 291 deletions(-) create mode 100644 src/main/java/com/_4paradigm/openmldb/featureplatform/utils/OpenmldbSqlUtil.java diff --git a/frontend/src/components/featureservice/FeatureServiceDetail.vue b/frontend/src/components/featureservice/FeatureServiceDetail.vue index 29be145..8138de9 100644 --- a/frontend/src/components/featureservice/FeatureServiceDetail.vue +++ b/frontend/src/components/featureservice/FeatureServiceDetail.vue @@ -30,7 +30,7 @@ + + \ No newline at end of file + }; + \ No newline at end of file diff --git a/frontend/src/components/featureservice/RequestFeatureService.vue b/frontend/src/components/featureservice/RequestFeatureService.vue index 3546f3a..9572573 100644 --- a/frontend/src/components/featureservice/RequestFeatureService.vue +++ b/frontend/src/components/featureservice/RequestFeatureService.vue @@ -27,25 +27,6 @@ -
diff --git a/frontend/src/components/featureservice/RequestModePage.vue b/frontend/src/components/featureservice/RequestModePage.vue index f10ef8b..65d3f39 100644 --- a/frontend/src/components/featureservice/RequestModePage.vue +++ b/frontend/src/components/featureservice/RequestModePage.vue @@ -3,11 +3,10 @@

- {{ name }} {{ version }} - + {{ featureNames }} {{ requestSchema }} {{ requestDemoData }} @@ -87,6 +86,7 @@ export default { data() { return { + featureNames: null, isUseJsonTestData: false, @@ -111,6 +111,22 @@ export default { methods: { init() { + axios.get(`/api/featureservices/${this.name}/${this.version}`) + .then(response => { + this.featureNames = response.data.featureNames; + + }) + .catch(error => { + var errorMessage = error.message; + if (error.response && error.response.data) { + errorMessage = error.response.data; + } + notification["error"]({ + message: this.$t('Execute Fail'), + description: errorMessage + }); + }) + axios.get(`/api/featureservices/${this.name}/${this.version}/request/schema`) .then(response => { this.requestSchema = response.data; @@ -152,7 +168,6 @@ export default { }); }, - submitForm() { var requestJson = {} diff --git a/frontend/src/locales/en.json b/frontend/src/locales/en.json index 0e521e2..7bbdb9c 100644 --- a/frontend/src/locales/en.json +++ b/frontend/src/locales/en.json @@ -255,6 +255,8 @@ "Refresh Status": "Refresh Status", "Filter SQL": "Filter SQL", "Add Index": "Add Index", - "Request Feature Service": "Request Feature Service" + "Request Feature Service": "Request Feature Service", + "Choose Index": "Choose Index", + "Online Query Feature Result": "Online Query Feature Result" } \ No newline at end of file diff --git a/frontend/src/locales/zh.json b/frontend/src/locales/zh.json index 94a6a45..e046ef1 100644 --- a/frontend/src/locales/zh.json +++ b/frontend/src/locales/zh.json @@ -257,6 +257,8 @@ "Refresh Status": "刷新状态", "Filter SQL": "过滤数据 SQL", "Add Index": "添加索引", - "Request Feature Service": "请求特征服务" + "Request Feature Service": "请求特征服务", + "Choose Index": "选择索引", + "Online Query Feature Result": "在线查询特征结果" } \ No newline at end of file diff --git a/src/main/java/com/_4paradigm/openmldb/featureplatform/controller/FeatureServiceController.java b/src/main/java/com/_4paradigm/openmldb/featureplatform/controller/FeatureServiceController.java index 2810cfd..9e49db1 100644 --- a/src/main/java/com/_4paradigm/openmldb/featureplatform/controller/FeatureServiceController.java +++ b/src/main/java/com/_4paradigm/openmldb/featureplatform/controller/FeatureServiceController.java @@ -152,6 +152,17 @@ public ResponseEntity requestFeatureService(@PathVariable String name, @ } } + @PostMapping(value = "/{name}/{version}/request/onlinequerymode", consumes = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity>> requestOnlineQueryMode(@PathVariable String name, @PathVariable String version, @RequestBody String dataRequest) throws SQLException { + try { + //tobe1 + return ResponseEntity.ok(featureServiceService.requestOnlineQueryMode(name, version, dataRequest)); + } catch (Exception e) { + logger.info(String.format("Call requestOnlineQueryMode with %s, %s and %s but get exception: %s", name, version, dataRequest, e.getMessage())); + throw new SQLException(e.getMessage()); + } + } + @GetMapping("/{name}/{version}/request/schema") public ResponseEntity getRequestSchema(@PathVariable String name, @PathVariable String version) throws SQLException { try { @@ -257,4 +268,15 @@ public ResponseEntity getOutputSchema(@PathVariable String name, @PathVa } } + @GetMapping("/{name}/{version}/indexes") + public ResponseEntity> getIndexNames(@PathVariable String name, @PathVariable String version) throws SQLException { + try { + return new ResponseEntity<>(featureServiceService.getIndexNames(name, version), HttpStatus.OK); + } catch (SQLException e) { + logger.info(String.format("Call getRequestSchema with %s and %s but get exception: %s", name, version, e.getMessage())); + throw new SQLException(e.getMessage()); + } + } + + } \ No newline at end of file diff --git a/src/main/java/com/_4paradigm/openmldb/featureplatform/service/FeatureServiceService.java b/src/main/java/com/_4paradigm/openmldb/featureplatform/service/FeatureServiceService.java index e8e6296..d3709aa 100644 --- a/src/main/java/com/_4paradigm/openmldb/featureplatform/service/FeatureServiceService.java +++ b/src/main/java/com/_4paradigm/openmldb/featureplatform/service/FeatureServiceService.java @@ -3,9 +3,16 @@ import com._4paradigm.openmldb.common.Pair; import com._4paradigm.openmldb.featureplatform.dao.model.*; import com._4paradigm.openmldb.featureplatform.utils.*; +import com._4paradigm.openmldb.jdbc.SQLResultSet; +import com._4paradigm.openmldb.proto.Common; +import com._4paradigm.openmldb.proto.NS; import com._4paradigm.openmldb.sdk.Column; import com._4paradigm.openmldb.sdk.Schema; import com._4paradigm.openmldb.sdk.impl.SqlClusterExecutor; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; @@ -22,6 +29,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Types; import java.util.*; @Repository @@ -356,32 +364,100 @@ public ResponseEntity requestFeatureService(String name, String version, return new ResponseEntity(responseBody, HttpStatus.valueOf(statusCode)); } - public static String removeDeploySubstring(String inputSql) { - int substringIndex = inputSql.toLowerCase().indexOf("select"); + public List> requestOnlineQueryMode(String name, String version, String requestData) throws IOException, SQLException { + // TODO: Get the db from feature service + FeatureServiceService featureServiceService = new FeatureServiceService(sqlExecutor); + FeatureService featureService = featureServiceService.getFeatureServiceByNameAndVersion(name, version); + String db = featureService.getDb(); + String deploymentSql = featureService.getSql(); - if (substringIndex >= 0) { - return inputSql.substring(substringIndex); - } else { - return inputSql; + String querySql = OpenmldbSqlUtil.removeDeployFromSql(deploymentSql); + + Schema schema = OpenmldbTableUtil.getMainTableSchema(sqlExecutor, querySql, db); + + Gson gson = new Gson(); + JsonObject jsonObject = gson.fromJson(requestData, JsonObject.class); + JsonArray dataJsonArray = jsonObject.getAsJsonArray("data"); + JsonArray indexJsonArray = jsonObject.getAsJsonArray("index"); + + // TODO: Support only one index now + String indexName = indexJsonArray.get(0).getAsString(); + String indexDataValue = dataJsonArray.getAsJsonArray().get(0).getAsString(); + + boolean isIndexStringType = false; + for (Column column: schema.getColumnList()) { + if (column.getColumnName().equals(indexName)) { + if(column.getSqlType() == Types.CHAR || column.getSqlType() == Types.VARCHAR) { + isIndexStringType = true; + } + } } + + String finalSql = String.format("%s WHERE %s = %s", querySql, indexName, indexDataValue); + if (isIndexStringType) { + // TODO: Handle single quote and double quote + finalSql = String.format("%s WHERE %s = '%s'", querySql, indexName, indexDataValue); + } + + Statement statement = sqlExecutor.getStatement(); + statement.execute("SET @@execute_mode='online'"); + statement.execute(String.format("USE %s", db)); + + statement.execute(finalSql); + + List> returnList = new ArrayList<>(); + + // TODO: Check if has result set + SQLResultSet resultSet = (SQLResultSet) statement.getResultSet(); + returnList = ResultSetUtil.resultSetToStringArray(resultSet); + resultSet.close(); + + return returnList; + } + public List getIndexNames(String name, String version) throws SQLException { + + FeatureServiceService featureServiceService = new FeatureServiceService(sqlExecutor); + FeatureService featureService = featureServiceService.getFeatureServiceByNameAndVersion(name, version); + String db = featureService.getDb(); + String deploymentSql = featureService.getSql(); + + String querySql = OpenmldbSqlUtil.removeDeployFromSql(deploymentSql); + + String mainTableName = OpenmldbTableUtil.getMainTableName(sqlExecutor, querySql, db); + + NS.TableInfo tableInfo = sqlExecutor.getTableInfo(db, mainTableName); + + // For example: ["name", "name,age"] + List indexColumnNames = new ArrayList<>(); + + + for (Common.ColumnKey columnKey: tableInfo.getColumnKeyList()) { + if(columnKey.getFlag() == 0){ + List columnNameList = new ArrayList<>(); + for (String columnName: columnKey.getColNameList()) { + columnNameList.add(columnName); + } + + indexColumnNames.add(String.join(",", columnNameList)); + } + } + + return indexColumnNames; + } + + + public Schema getRequestSchema(String serviceName, String version) throws SQLException { FeatureService featureService = getFeatureServiceByNameAndVersion(serviceName, version); String sql = featureService.getSql(); String db = featureService.getDb(); - String selectSql = removeDeploySubstring(sql); - - List> tables = SqlClusterExecutor.getDependentTables(selectSql, db, - OpenmldbTableUtil.getSystemSchemaMaps(sqlExecutor)); - - Pair mainTablePair = tables.get(0); + String selectSql = OpenmldbSqlUtil.removeDeployFromSql(sql); - String mainDb = mainTablePair.getKey(); - String mainTable = mainTablePair.getValue(); - Schema schema = sqlExecutor.getTableSchema(mainDb, mainTable); + Schema schema = OpenmldbTableUtil.getMainTableSchema(sqlExecutor, selectSql, db); return schema; } @@ -439,7 +515,7 @@ public String getLatestVersion(String serviceName) throws SQLException { public List getDependentTables(String name, String version) throws SQLException { FeatureService featureService = getFeatureServiceByNameAndVersion(name, version); - String selectSql = removeDeploySubstring(featureService.getSql()); + String selectSql = OpenmldbSqlUtil.removeDeployFromSql(featureService.getSql()); List> tables = SqlClusterExecutor.getDependentTables(selectSql, featureService.getDb(), OpenmldbTableUtil.getSystemSchemaMaps(sqlExecutor)); @@ -461,7 +537,7 @@ public Schema getOutputSchema(String serviceName, String version) throws SQLExce String sql = featureService.getSql(); String db = featureService.getDb(); - String selectSql = removeDeploySubstring(sql); + String selectSql = OpenmldbSqlUtil.removeDeployFromSql(sql); Schema schema = SqlClusterExecutor.genOutputSchema(selectSql, db, OpenmldbTableUtil.getSystemSchemaMaps(sqlExecutor)); return schema; diff --git a/src/main/java/com/_4paradigm/openmldb/featureplatform/service/TableService.java b/src/main/java/com/_4paradigm/openmldb/featureplatform/service/TableService.java index c53423e..f92bd59 100644 --- a/src/main/java/com/_4paradigm/openmldb/featureplatform/service/TableService.java +++ b/src/main/java/com/_4paradigm/openmldb/featureplatform/service/TableService.java @@ -4,6 +4,7 @@ import com._4paradigm.openmldb.featureplatform.dao.model.FeatureService; import com._4paradigm.openmldb.featureplatform.dao.model.FeatureView; import com._4paradigm.openmldb.featureplatform.dao.model.SimpleTableInfo; +import com._4paradigm.openmldb.featureplatform.utils.OpenmldbSqlUtil; import com._4paradigm.openmldb.featureplatform.utils.OpenmldbTableUtil; import com._4paradigm.openmldb.sdk.Schema; import com._4paradigm.openmldb.sdk.impl.SqlClusterExecutor; @@ -63,7 +64,7 @@ public List getRelatedFeatureServices(String db, String table) t for (FeatureService featureService : allFeatureServices) { - String selectSql = FeatureServiceService.removeDeploySubstring(featureService.getSql()); + String selectSql = OpenmldbSqlUtil.removeDeployFromSql(featureService.getSql()); List> dependentTables = SqlClusterExecutor.getDependentTables(selectSql, featureService.getDb(), OpenmldbTableUtil.getSystemSchemaMaps(sqlExecutor)); @@ -117,4 +118,6 @@ public void deleteTable(String db, String table) throws SQLException { statement.close(); } + + } \ No newline at end of file diff --git a/src/main/java/com/_4paradigm/openmldb/featureplatform/utils/OpenmldbSqlUtil.java b/src/main/java/com/_4paradigm/openmldb/featureplatform/utils/OpenmldbSqlUtil.java new file mode 100644 index 0000000..6b77fdd --- /dev/null +++ b/src/main/java/com/_4paradigm/openmldb/featureplatform/utils/OpenmldbSqlUtil.java @@ -0,0 +1,30 @@ +package com._4paradigm.openmldb.featureplatform.utils; + +import com._4paradigm.openmldb.common.Pair; +import com._4paradigm.openmldb.sdk.Schema; +import com._4paradigm.openmldb.sdk.impl.SqlClusterExecutor; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class OpenmldbSqlUtil { + + public static String removeDeployFromSql(String inputSql) { + String[] parts = inputSql.trim().split(" "); + + // Using StringBuilder to concatenate strings efficiently + StringBuilder sb = new StringBuilder(); + for (int i = 2; i < parts.length; i++) { + sb.append(parts[i]); + // Add space back except for the last element + if (i < parts.length - 1) { + sb.append(" "); + } + } + + return sb.toString(); + } + +} \ No newline at end of file diff --git a/src/main/java/com/_4paradigm/openmldb/featureplatform/utils/OpenmldbTableUtil.java b/src/main/java/com/_4paradigm/openmldb/featureplatform/utils/OpenmldbTableUtil.java index 84d1255..89205ea 100644 --- a/src/main/java/com/_4paradigm/openmldb/featureplatform/utils/OpenmldbTableUtil.java +++ b/src/main/java/com/_4paradigm/openmldb/featureplatform/utils/OpenmldbTableUtil.java @@ -1,5 +1,6 @@ package com._4paradigm.openmldb.featureplatform.utils; +import com._4paradigm.openmldb.common.Pair; import com._4paradigm.openmldb.sdk.Schema; import com._4paradigm.openmldb.sdk.impl.SqlClusterExecutor; @@ -28,5 +29,27 @@ public static Map> getSystemSchemaMaps(SqlClusterExe return schemaMaps; } + public static Schema getMainTableSchema(SqlClusterExecutor sqlExecutor, String sql, String db) throws SQLException { + List> tables = SqlClusterExecutor.getDependentTables(sql, db, + OpenmldbTableUtil.getSystemSchemaMaps(sqlExecutor)); + + Pair mainTablePair = tables.get(0); + + String mainDb = mainTablePair.getKey(); + String mainTable = mainTablePair.getValue(); + + Schema schema = sqlExecutor.getTableSchema(mainDb, mainTable); + return schema; + } + + public static String getMainTableName(SqlClusterExecutor sqlExecutor, String sql, String db) throws SQLException { + List> tables = SqlClusterExecutor.getDependentTables(sql, db, + OpenmldbTableUtil.getSystemSchemaMaps(sqlExecutor)); + + Pair mainTablePair = tables.get(0); + + String mainTable = mainTablePair.getValue(); + return mainTable; + } } \ No newline at end of file From 57eab0a9e3c9b5906147fcc717b0e185e6a2b182 Mon Sep 17 00:00:00 2001 From: tobe Date: Thu, 4 Jan 2024 14:44:37 +0800 Subject: [PATCH 3/4] Add main table in online query mode --- .../featureservice/OnlineQueryModePage.vue | 49 ++++++++++++++++-- .../components/importdata/ImportDataMenu.vue | 3 +- frontend/src/locales/en.json | 3 +- frontend/src/locales/zh.json | 3 +- .../controller/FeatureServiceController.java | 11 ++++ .../dao/model/DatabaseTable.java | 18 +++++++ .../service/FeatureServiceService.java | 50 +++++++++++++------ .../featureplatform/service/SqlService.java | 2 +- .../utils/OpenmldbTableUtil.java | 10 ++++ 9 files changed, 124 insertions(+), 25 deletions(-) create mode 100644 src/main/java/com/_4paradigm/openmldb/featureplatform/dao/model/DatabaseTable.java diff --git a/frontend/src/components/featureservice/OnlineQueryModePage.vue b/frontend/src/components/featureservice/OnlineQueryModePage.vue index dad1bb2..bdd8e3c 100644 --- a/frontend/src/components/featureservice/OnlineQueryModePage.vue +++ b/frontend/src/components/featureservice/OnlineQueryModePage.vue @@ -1,4 +1,10 @@ \ No newline at end of file diff --git a/frontend/src/components/feature/FeatureDetail.vue b/frontend/src/components/feature/FeatureDetail.vue index 20aec82..f44dccc 100644 --- a/frontend/src/components/feature/FeatureDetail.vue +++ b/frontend/src/components/feature/FeatureDetail.vue @@ -7,8 +7,63 @@ {{ feature.featureName }} {{ feature.type}} {{ feature.description}} + + {{ featureSourceSql }} +
+ +
+ {{ $t('Display Sample Features') }} + + +
+
+ + + + + + + + + + + + +

{{ column.name }} ({{ column.type }}):

+ + + + +
+ + + {{ $t('Submit') }} + +
+
+ +
+
+

{{ $t('Feature Result') }}

+ + + +
+ diff --git a/frontend/src/components/featureservice/OnlineQueryModePage.vue b/frontend/src/components/featureservice/OnlineQueryModePage.vue index bdd8e3c..478ef5d 100644 --- a/frontend/src/components/featureservice/OnlineQueryModePage.vue +++ b/frontend/src/components/featureservice/OnlineQueryModePage.vue @@ -19,43 +19,49 @@
- - + 随机生成特征 +  通过索引过滤特征 - - - - - - - - - -

{{ column.name }} ({{ column.type }}):

- +
+
+ + + - + :label="$t('Choose Index')"> + + + + - - - - {{ $t('Submit') }} - - - + + + +

{{ column.name }} ({{ column.type }}):

+ + + + +
+ + + {{ $t('Submit') }} + + +
+

-

{{ $t('Online Query Feature Result') }}

+

{{ $t('Feature Result') }}