From 6336844c8c0632d8385664224b9fd28c76e90635 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 29 May 2024 20:45:18 +0800 Subject: [PATCH 01/25] [GIE Compiler] support reading ir meta from meta service by http api --- .../compiler/conf/ir.compiler.properties | 9 ++ interactive_engine/compiler/graph.yaml | 93 ++++++++++++++++++ interactive_engine/compiler/graph_test.json | 1 + .../com/alibaba/graphscope/GraphServer.java | 27 ++++-- .../com/alibaba/graphscope/common/IrPlan.java | 2 +- .../channel/MetaServiceChannelFetcher.java | 47 +++++++++ .../common/config/FrontendConfig.java | 9 ++ .../graphscope/common/config/GraphConfig.java | 14 --- .../graphscope/common/config/YamlConfigs.java | 21 ---- .../graphscope/common/ir/meta/IrMeta.java | 70 ++++++++++++++ .../common/ir/meta/IrMetaUpdater.java | 24 +++++ .../graphscope/common/ir/meta/SnapshotId.java | 41 ++++++++ .../ir/meta/fetcher/DynamicIrMetaFetcher.java | 76 +++++++++++++++ .../common/ir/meta/fetcher/IrMetaFetcher.java | 39 ++++++++ .../ir/meta/fetcher/StaticIrMetaFetcher.java | 39 ++++++++ .../meta/procedure/GraphStoredProcedures.java | 74 +++++++++++--- .../ir/meta/reader/HttpIrMetaReader.java | 96 +++++++++++++++++++ .../IrMetaReader.java} | 13 ++- .../ir/meta/reader/LocalIrMetaReader.java | 60 ++++++++++++ .../ir/meta/reader/LocalMetaDataReader.java | 76 --------------- .../common/ir/meta/reader/MetaDataReader.java | 30 ------ .../common/ir/meta/schema/IrGraphSchema.java | 5 +- .../common/ir/planner/GraphIOProcessor.java | 2 +- .../ir/runtime/ffi/FfiPhysicalBuilder.java | 2 +- .../proto/GraphRelProtoPhysicalBuilder.java | 2 +- .../common/ir/tools/GraphPlanner.java | 9 +- .../common/ir/tools/LogicalPlanFactory.java | 2 +- .../common/ir/tools/QueryCache.java | 2 +- .../common/manager/IrMetaQueryCallback.java | 4 +- .../common/store/ExperimentalMetaFetcher.java | 37 ------- .../graphscope/common/store/IrMeta.java | 63 ------------ .../common/store/IrMetaFetcher.java | 23 ----- .../graphscope/common/store/SnapshotId.java | 39 -------- .../antlr4/visitor/LogicalPlanVisitor.java | 2 +- .../antlr4/visitor/ProcedureCallVisitor.java | 6 +- .../cypher/executor/GraphQueryExecutor.java | 2 +- .../processor/IrTestOpProcessor.java | 2 +- .../processor/IrStandardOpProcessor.java | 2 +- .../plugin/processor/LifeCycleSupplier.java | 2 +- .../GremlinCalciteScriptEngineFactory.java | 2 +- .../common/config/YamlConfigTest.java | 10 +- .../graphscope/common/ir/QueryCacheTest.java | 2 +- .../alibaba/graphscope/common/ir/Utils.java | 23 ++--- .../common/ir/planner/cbo/BITest.java | 2 +- .../common/ir/planner/cbo/CBOTest.java | 2 +- .../common/ir/planner/cbo/LdbcTest.java | 2 +- .../common/ir/planner/cbo/STPathTest.java | 2 +- .../common/ir/runtime/FfiLogicalPlanTest.java | 2 +- .../ir/runtime/GraphRelToProtoTest.java | 2 +- .../cypher/antlr4/CallProcedureTest.java | 6 +- .../graphscope/cypher/antlr4/Utils.java | 9 ++ .../alibaba/graphscope/frontend/Frontend.java | 5 +- .../frontend/JsonFileSchemaFetcher.java | 55 ----------- .../frontend/VineyardIrMetaReader.java | 48 ++++++++++ .../frontend/VineyardMetaFetcher.java | 46 --------- .../servers/ir/FrontendQueryManager.java | 4 +- ...etaFetcher.java => GrootIrMetaReader.java} | 33 +++---- .../groot/servers/ir/IrServiceProducer.java | 8 +- 58 files changed, 820 insertions(+), 510 deletions(-) create mode 100644 interactive_engine/compiler/graph.yaml create mode 100644 interactive_engine/compiler/graph_test.json create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaUpdater.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/SnapshotId.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/IrMetaFetcher.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java rename interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/{procedure/StoredProcedures.java => reader/IrMetaReader.java} (65%) create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java delete mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java delete mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/MetaDataReader.java delete mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/ExperimentalMetaFetcher.java delete mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMeta.java delete mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMetaFetcher.java delete mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/SnapshotId.java delete mode 100644 interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/JsonFileSchemaFetcher.java create mode 100644 interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java delete mode 100644 interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardMetaFetcher.java rename interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/{GrootMetaFetcher.java => GrootIrMetaReader.java} (56%) diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties index 7f18a0271721..c19910f97cf7 100644 --- a/interactive_engine/compiler/conf/ir.compiler.properties +++ b/interactive_engine/compiler/conf/ir.compiler.properties @@ -61,3 +61,12 @@ calcite.default.charset: UTF-8 # set the max capacity of the result streaming buffer for each query # per.query.stream.buffer.max.capacity: 256 + +# set the mode to read ir meta, either from a 'local' file or via remote 'http'. +# ir.meta.reader.mode: local + +# if ir.meta.reader.mode is 'http', set the remote ir meta service host +# ir.meta.service.host: localhost:8080 + +# if ir.meta.reader.mode is 'http', set the interval in milliseconds to fetch ir meta +# ir.meta.fetch.interval.ms: 1000 diff --git a/interactive_engine/compiler/graph.yaml b/interactive_engine/compiler/graph.yaml new file mode 100644 index 000000000000..85a94f9077c1 --- /dev/null +++ b/interactive_engine/compiler/graph.yaml @@ -0,0 +1,93 @@ +name: modern # then must have a modern dir under ${data} directory +store_type: mutable_csr # v6d, groot, gart +stored_procedures: + - name: ldbc_ic2 + description: "A stored procedures for the ldbc complex interactive workload 2" + type: x_cypher + params: + - name: personId2 + type: + primitive_type: DT_SIGNED_INT64 + - name: maxDate + type: + primitive_type: DT_SIGNED_INT64 + returns: + - name: name + type: + string: + long_text: + query: "MATCH(n: PERSON ${personId2}) WHERE n.creationDate < ${maxDate} RETURN n.firstName AS name LIMIT 10;" + library: libquery_ic2.so + encoding: string +schema: + vertex_types: + - type_name: person + type_id: 0 + x_csr_params: + max_vertex_num: 100 + properties: + - property_id: 0 + property_name: id + property_type: + primitive_type: DT_SIGNED_INT64 + - property_id: 1 + property_name: name + property_type: + string: + long_text: + - property_id: 2 + property_name: age + property_type: + primitive_type: DT_SIGNED_INT32 + primary_keys: + - id + - type_name: software + type_id: 1 + x_csr_params: + max_vertex_num: 100 + properties: + - property_id: 0 + property_name: id + property_type: + primitive_type: DT_SIGNED_INT64 + x_csr_params: + - property_id: 1 + property_name: name + property_type: + string: + long_text: + - property_id: 2 + property_name: lang + property_type: + string: + long_text: + - property_id: 3 + property_name: creationDate + property_type: + temporal: + date32: + primary_keys: + - id + edge_types: + - type_name: knows + type_id: 2 + vertex_type_pair_relations: + - source_vertex: person + destination_vertex: person + relation: MANY_TO_MANY + properties: + - property_id: 0 + property_name: weight + property_type: + primitive_type: DT_DOUBLE + - type_name: created + type_id: 3 + vertex_type_pair_relations: + - source_vertex: person + destination_vertex: software + relation: MANY_TO_MANY + properties: + - property_id: 0 + property_name: weight + property_type: + primitive_type: DT_DOUBLE diff --git a/interactive_engine/compiler/graph_test.json b/interactive_engine/compiler/graph_test.json new file mode 100644 index 000000000000..606d2dac7c11 --- /dev/null +++ b/interactive_engine/compiler/graph_test.json @@ -0,0 +1 @@ +{"bolt_port":7687,"graph":{"creation_time":1716969975953,"data_update_time":1716969975953,"description":"","id":"1","name":"ldbc_snb","schema":{"edge_types":[{"type_id":0,"type_name":"HASCREATOR","vertex_type_pair_relations":[{"destination_vertex":"PERSON","relation":"MANY_TO_ONE","source_vertex":"COMMENT"},{"destination_vertex":"PERSON","relation":"MANY_TO_ONE","source_vertex":"POST"}]},{"type_id":1,"type_name":"HASTAG","vertex_type_pair_relations":[{"destination_vertex":"TAG","relation":"MANY_TO_MANY","source_vertex":"POST"},{"destination_vertex":"TAG","relation":"MANY_TO_MANY","source_vertex":"FORUM"},{"destination_vertex":"TAG","relation":"MANY_TO_MANY","source_vertex":"COMMENT"}]},{"type_id":2,"type_name":"REPLYOF","vertex_type_pair_relations":[{"destination_vertex":"COMMENT","relation":"MANY_TO_MANY","source_vertex":"COMMENT"},{"destination_vertex":"POST","relation":"MANY_TO_ONE","source_vertex":"COMMENT"}]},{"type_id":3,"type_name":"CONTAINEROF","vertex_type_pair_relations":[{"destination_vertex":"POST","relation":"ONE_TO_MANY","source_vertex":"FORUM"}]},{"properties":[{"property_id":0,"property_name":"joinDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}}],"type_id":4,"type_name":"HASMEMBER","vertex_type_pair_relations":[{"destination_vertex":"PERSON","relation":"MANY_TO_MANY","source_vertex":"FORUM"}]},{"type_id":5,"type_name":"HASMODERATOR","vertex_type_pair_relations":[{"destination_vertex":"PERSON","relation":"MANY_TO_ONE","source_vertex":"FORUM"}]},{"type_id":6,"type_name":"HASINTEREST","vertex_type_pair_relations":[{"destination_vertex":"TAG","relation":"MANY_TO_MANY","source_vertex":"PERSON"}]},{"type_id":7,"type_name":"ISLOCATEDIN","vertex_type_pair_relations":[{"destination_vertex":"PLACE","relation":"MANY_TO_ONE","source_vertex":"COMMENT"},{"destination_vertex":"PLACE","relation":"MANY_TO_ONE","source_vertex":"PERSON"},{"destination_vertex":"PLACE","relation":"MANY_TO_ONE","source_vertex":"POST"},{"destination_vertex":"PLACE","relation":"MANY_TO_ONE","source_vertex":"ORGANISATION"}]},{"properties":[{"property_id":0,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}}],"type_id":8,"type_name":"KNOWS","vertex_type_pair_relations":[{"destination_vertex":"PERSON","relation":"MANY_TO_MANY","source_vertex":"PERSON"}]},{"properties":[{"property_id":0,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}}],"type_id":9,"type_name":"LIKES","vertex_type_pair_relations":[{"destination_vertex":"COMMENT","relation":"MANY_TO_MANY","source_vertex":"PERSON"},{"destination_vertex":"POST","relation":"MANY_TO_MANY","source_vertex":"PERSON"}]},{"properties":[{"property_id":0,"property_name":"workFrom","property_type":{"primitive_type":"DT_SIGNED_INT32"}}],"type_id":10,"type_name":"WORKAT","vertex_type_pair_relations":[{"destination_vertex":"ORGANISATION","relation":"MANY_TO_MANY","source_vertex":"PERSON"}]},{"type_id":11,"type_name":"ISPARTOF","vertex_type_pair_relations":[{"destination_vertex":"PLACE","relation":"MANY_TO_MANY","source_vertex":"PLACE"}]},{"type_id":12,"type_name":"HASTYPE","vertex_type_pair_relations":[{"destination_vertex":"TAGCLASS","relation":"MANY_TO_ONE","source_vertex":"TAG"}]},{"type_id":13,"type_name":"ISSUBCLASSOF","vertex_type_pair_relations":[{"destination_vertex":"TAGCLASS","relation":"MANY_TO_MANY","source_vertex":"TAGCLASS"}]},{"properties":[{"property_id":0,"property_name":"classYear","property_type":{"primitive_type":"DT_SIGNED_INT32"}}],"type_id":14,"type_name":"STUDYAT","vertex_type_pair_relations":[{"destination_vertex":"ORGANISATION","relation":"MANY_TO_MANY","source_vertex":"PERSON"}]}],"vertex_types":[{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"name","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"url","property_type":{"string":{"long_text":null}}},{"property_id":3,"property_name":"type","property_type":{"string":{"long_text":null}}}],"type_id":0,"type_name":"PLACE","x_csr_params":{"max_vertex_num":1460}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"firstName","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"lastName","property_type":{"string":{"long_text":null}}},{"property_id":3,"property_name":"gender","property_type":{"string":{"long_text":null}}},{"property_id":4,"property_name":"birthday","property_type":{"temporal":{"timestamp":null}}},{"property_id":5,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":6,"property_name":"locationIP","property_type":{"string":{"long_text":null}}},{"property_id":7,"property_name":"browserUsed","property_type":{"string":{"long_text":null}}},{"property_id":8,"property_name":"language","property_type":{"string":{"long_text":null}}},{"property_id":9,"property_name":"email","property_type":{"string":{"long_text":null}}}],"type_id":1,"type_name":"PERSON","x_csr_params":{"max_vertex_num":1254000}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":2,"property_name":"locationIP","property_type":{"string":{"long_text":null}}},{"property_id":3,"property_name":"browserUsed","property_type":{"string":{"long_text":null}}},{"property_id":4,"property_name":"content","property_type":{"string":{"long_text":null}}},{"property_id":5,"property_name":"length","property_type":{"primitive_type":"DT_SIGNED_INT32"}}],"type_id":2,"type_name":"COMMENT","x_csr_params":{"max_vertex_num":762949871}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"imageFile","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":3,"property_name":"locationIP","property_type":{"string":{"long_text":null}}},{"property_id":4,"property_name":"browserUsed","property_type":{"string":{"long_text":null}}},{"property_id":5,"property_name":"language","property_type":{"string":{"long_text":null}}},{"property_id":6,"property_name":"content","property_type":{"string":{"long_text":null}}},{"property_id":7,"property_name":"length","property_type":{"primitive_type":"DT_SIGNED_INT32"}}],"type_id":3,"type_name":"POST","x_csr_params":{"max_vertex_num":187893047}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"title","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}}],"type_id":4,"type_name":"FORUM","x_csr_params":{"max_vertex_num":12549398}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"type","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"name","property_type":{"string":{"long_text":null}}},{"property_id":3,"property_name":"url","property_type":{"string":{"long_text":null}}}],"type_id":5,"type_name":"ORGANISATION","x_csr_params":{"max_vertex_num":7955}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"name","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"url","property_type":{"string":{"long_text":null}}}],"type_id":6,"type_name":"TAGCLASS","x_csr_params":{"max_vertex_num":71}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"name","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"url","property_type":{"string":{"long_text":null}}}],"type_id":7,"type_name":"TAG","x_csr_params":{"max_vertex_num":16080}}]},"store_type":"mutable_csr","stored_procedures":[]},"gremlin_port":8182,"hqps_port":10000,"start_time":1716974357943,"status":"Stopped"} \ No newline at end of file diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java index f5ee93fd4a9c..a016c94f333c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java @@ -20,14 +20,18 @@ import com.alibaba.graphscope.common.client.channel.ChannelFetcher; import com.alibaba.graphscope.common.client.channel.HostURIChannelFetcher; import com.alibaba.graphscope.common.client.channel.HostsRpcChannelFetcher; +import com.alibaba.graphscope.common.client.channel.MetaServiceChannelFetcher; import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.config.GraphConfig; -import com.alibaba.graphscope.common.ir.meta.reader.LocalMetaDataReader; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.fetcher.DynamicIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.reader.HttpIrMetaReader; +import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader; import com.alibaba.graphscope.common.ir.tools.*; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; -import com.alibaba.graphscope.common.store.ExperimentalMetaFetcher; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.cypher.antlr4.parser.CypherAntlr4Parser; import com.alibaba.graphscope.cypher.antlr4.visitor.LogicalPlanVisitor; import com.alibaba.graphscope.cypher.service.CypherBootstrapper; @@ -164,15 +168,26 @@ public static void main(String[] args) throws Exception { throw new IllegalArgumentException("usage: GraphServer ''"); } Configs configs = Configs.Factory.create(args[0]); - IrMetaQueryCallback queryCallback = - new IrMetaQueryCallback( - new ExperimentalMetaFetcher(new LocalMetaDataReader(configs))); + IrMetaQueryCallback queryCallback = new IrMetaQueryCallback(createIrMetaFetcher(configs)); GraphServer server = new GraphServer( configs, getChannelFetcher(configs), queryCallback, getTestGraph(configs)); server.start(); } + private static IrMetaFetcher createIrMetaFetcher(Configs configs) throws IOException { + String readerMode = FrontendConfig.IR_META_READER_MODE.get(configs); + switch (readerMode) { + case "local": + return new StaticIrMetaFetcher(new LocalIrMetaReader(configs)); + case "http": + return new DynamicIrMetaFetcher( + new HttpIrMetaReader(new MetaServiceChannelFetcher(configs)), configs); + default: + throw new IllegalArgumentException("unknown ir meta reader mode: " + readerMode); + } + } + private static GraphProperties getTestGraph(Configs configs) { GraphProperties testGraph; switch (GraphConfig.GRAPH_STORE.get(configs)) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java index 29e15ff49912..600cedab8f31 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java @@ -27,9 +27,9 @@ import com.alibaba.graphscope.common.intermediate.process.SinkArg; import com.alibaba.graphscope.common.intermediate.process.SinkByColumns; import com.alibaba.graphscope.common.intermediate.process.SinkGraph; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.jna.IrCoreLibrary; import com.alibaba.graphscope.common.jna.type.*; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.common.utils.ClassUtils; import com.alibaba.graphscope.gaia.proto.Common; import com.alibaba.graphscope.gaia.proto.GraphAlgebra; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java new file mode 100644 index 000000000000..531fb1d2906b --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java @@ -0,0 +1,47 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.client.channel; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.FrontendConfig; +import com.google.common.collect.ImmutableList; + +import java.net.URI; +import java.util.List; + +// read meta service host from config and return it as URI format +public class MetaServiceChannelFetcher implements ChannelFetcher { + private static final String schema = "http"; + private Configs graphConfig; + + public MetaServiceChannelFetcher(Configs graphConfig) { + this.graphConfig = graphConfig; + } + + @Override + public List fetch() { + String host = FrontendConfig.IR_META_SERVICE_HOST.get(graphConfig); + return ImmutableList.of(URI.create(schema + "://" + host)); + } + + @Override + public Type getType() { + return Type.HTTP; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java index b4add388a962..91411eb48c91 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java @@ -60,4 +60,13 @@ public class FrontendConfig { public static final Config PER_QUERY_STREAM_BUFFER_MAX_CAPACITY = Config.intConfig("per.query.stream.buffer.max.capacity", 256); + + public static final Config IR_META_FETCH_INTERVAL_MS = + Config.longConfig("ir.meta.fetch.interval.ms", 1000); + + public static Config IR_META_READER_MODE = + Config.stringConfig("ir.meta.reader.mode", "local"); + + public static Config IR_META_SERVICE_HOST = + Config.stringConfig("ir.meta.service.host", "localhost:8080"); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java index 53226b72170c..1b625f9e5549 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java @@ -19,18 +19,4 @@ public class GraphConfig { public static final Config GRAPH_SCHEMA = Config.stringConfig("graph.schema", "."); public static final Config GRAPH_STORE = Config.stringConfig("graph.store", "exp"); - - @Deprecated - public static final Config GRAPH_STORED_PROCEDURES = - Config.stringConfig("graph.stored.procedures", ""); - - @Deprecated - public static final Config GRAPH_STORED_PROCEDURES_ENABLE_LISTS = - Config.stringConfig("graph.stored.procedures.enable.lists", ""); - - // denote stored procedures in yaml format, refer to test resource file - // 'config/modern/graph.yaml' for more info about - // the format - public static final Config GRAPH_STORED_PROCEDURES_YAML = - Config.stringConfig("graph.stored.procedures.yaml", ""); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index 3e9c79f07b4c..55f481acbf47 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -22,7 +22,6 @@ import org.apache.commons.lang3.StringUtils; import org.yaml.snakeyaml.Yaml; -import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -75,26 +74,6 @@ public class YamlConfigs extends Configs { return "cpp-mcsr"; } }) - .put( - "graph.stored.procedures.yaml", - (Configs configs) -> { - File schemaFile = new File(GraphConfig.GRAPH_SCHEMA.get(configs)); - if (!schemaFile.exists() || !schemaFile.getName().endsWith(".yaml")) { - return null; - } - try { - Yaml yaml = new Yaml(); - Map yamlAsMap = - yaml.load(new FileInputStream(schemaFile)); - Object value; - if ((value = yamlAsMap.get("stored_procedures")) == null) { - return null; - } - return yaml.dump(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - }) .put( "pegasus.worker.num", (Configs configs) -> { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java new file mode 100644 index 000000000000..eae98a502418 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java @@ -0,0 +1,70 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta; + +import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; +import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.io.IOException; +import java.util.Objects; + +/** + * This class describes all the metadata information required by IR, including Schema, Procedures, and Statistics. + * They are versioned using the same snapshot id. + */ +public class IrMeta { + private final SnapshotId snapshotId; + private final IrGraphSchema schema; + private final @Nullable GraphStoredProcedures storedProcedures; + + public IrMeta(IrGraphSchema schema) throws IOException { + this(SnapshotId.createEmpty(), schema); + } + + public IrMeta(IrGraphSchema schema, GraphStoredProcedures storedProcedures) { + this(SnapshotId.createEmpty(), schema, storedProcedures); + } + + public IrMeta(SnapshotId snapshotId, IrGraphSchema schema) throws IOException { + this.snapshotId = Objects.requireNonNull(snapshotId); + this.schema = Objects.requireNonNull(schema); + this.storedProcedures = null; + } + + public IrMeta( + SnapshotId snapshotId, IrGraphSchema schema, GraphStoredProcedures storedProcedures) { + this.snapshotId = Objects.requireNonNull(snapshotId); + this.schema = Objects.requireNonNull(schema); + this.storedProcedures = Objects.requireNonNull(storedProcedures); + } + + public IrGraphSchema getSchema() { + return schema; + } + + public SnapshotId getSnapshotId() { + return snapshotId; + } + + public @Nullable GraphStoredProcedures getStoredProcedures() { + return storedProcedures; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaUpdater.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaUpdater.java new file mode 100644 index 000000000000..44ebf33e10b8 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaUpdater.java @@ -0,0 +1,24 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta; + +// This interface triggers the update of Meta Data +public interface IrMetaUpdater { + IrMeta onUpdate(); +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/SnapshotId.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/SnapshotId.java new file mode 100644 index 000000000000..147ea59a4693 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/SnapshotId.java @@ -0,0 +1,41 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta; + +public class SnapshotId { + public static final SnapshotId createEmpty() { + return new SnapshotId(false, -1); + } + + private final boolean acquired; + private final long id; + + public SnapshotId(boolean acquired, long id) { + this.acquired = acquired; + this.id = id; + } + + public boolean isAcquired() { + return acquired; + } + + public long getId() { + return id; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java new file mode 100644 index 000000000000..c3e82bfac8aa --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java @@ -0,0 +1,76 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta.fetcher; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.FrontendConfig; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.IrMetaUpdater; +import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; +import com.alibaba.graphscope.common.ir.meta.reader.IrMetaReader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Periodically update IrMeta, with the update frequency controlled by configuration. + * Specifically, for procedures, a remote update will be actively triggered when they are not found locally. + */ +public class DynamicIrMetaFetcher extends IrMetaFetcher implements IrMetaUpdater { + private static final Logger logger = LoggerFactory.getLogger(DynamicIrMetaFetcher.class); + private final ScheduledExecutorService scheduler; + private volatile IrMeta currentState; + + public DynamicIrMetaFetcher(IrMetaReader dataReader, Configs configs) { + super(dataReader); + this.scheduler = new ScheduledThreadPoolExecutor(1); + this.scheduler.scheduleAtFixedRate( + () -> onUpdate(), + 0, + FrontendConfig.IR_META_FETCH_INTERVAL_MS.get(configs), + TimeUnit.MILLISECONDS); + } + + @Override + public Optional fetch() { + return currentState == null ? Optional.empty() : Optional.of(currentState); + } + + @Override + public synchronized IrMeta onUpdate() { + try { + this.currentState = this.reader.readMeta(); + if (this.currentState != null) { + GraphStoredProcedures procedures = this.currentState.getStoredProcedures(); + if (procedures != null) { + procedures.registerIrMetaUpdater(this); + } + } + } catch (IOException e) { + logger.warn("failed to update meta data, error is {}", e); + } + return this.currentState; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/IrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/IrMetaFetcher.java new file mode 100644 index 000000000000..695a6c3c7f67 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/IrMetaFetcher.java @@ -0,0 +1,39 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta.fetcher; + +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.reader.IrMetaReader; + +import java.util.Optional; + +/** + * This interface primarily describes the strategies to obtaining IrMeta, which are mainly of two types: static and dynamic. + * The static strategy {@link StaticIrMetaFetcher} assumes that IrMeta does not change after initialization, + * while the dynamic strategy {@link DynamicIrMetaFetcher} assumes that IrMeta can change. + */ +public abstract class IrMetaFetcher { + protected IrMetaReader reader; + + protected IrMetaFetcher(IrMetaReader reader) { + this.reader = reader; + } + + public abstract Optional fetch(); +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java new file mode 100644 index 000000000000..46f29ebc99bd --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java @@ -0,0 +1,39 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta.fetcher; + +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.reader.IrMetaReader; + +import java.io.IOException; +import java.util.Optional; + +public class StaticIrMetaFetcher extends IrMetaFetcher { + private final IrMeta meta; + + public StaticIrMetaFetcher(IrMetaReader dataReader) throws IOException { + super(dataReader); + this.meta = this.reader.readMeta(); + } + + @Override + public Optional fetch() { + return Optional.of(this.meta); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/GraphStoredProcedures.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/GraphStoredProcedures.java index badc9894fca2..f2d306264d81 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/GraphStoredProcedures.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/GraphStoredProcedures.java @@ -16,32 +16,78 @@ package com.alibaba.graphscope.common.ir.meta.procedure; -import com.alibaba.graphscope.common.ir.meta.reader.MetaDataReader; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.IrMetaUpdater; import com.google.common.collect.Maps; +import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; -public class GraphStoredProcedures implements StoredProcedures { - private static Logger logger = LoggerFactory.getLogger(GraphStoredProcedures.class); +public class GraphStoredProcedures { private final Map storedProcedureMetaMap; + private IrMetaUpdater metaUpdater; - public GraphStoredProcedures(MetaDataReader reader) throws Exception { - this.storedProcedureMetaMap = Maps.newLinkedHashMap(); - for (InputStream inputStream : reader.getStoredProcedures()) { - StoredProcedureMeta createdMeta = StoredProcedureMeta.Deserializer.perform(inputStream); - this.storedProcedureMetaMap.put(createdMeta.getName(), createdMeta); - logger.debug("Got stored procedure: {} from reader", createdMeta.getName()); - inputStream.close(); + public GraphStoredProcedures(InputStream metaStream) { + Yaml yaml = new Yaml(); + Map yamlAsMap = yaml.load(metaStream); + List procedureList = (List) yamlAsMap.get("stored_procedures"); + if (ObjectUtils.isEmpty(procedureList)) { + this.storedProcedureMetaMap = Maps.newLinkedHashMap(); + } else { + this.storedProcedureMetaMap = + procedureList.stream() + .map( + k -> { + try { + String procedureYaml = yaml.dump(k); + return StoredProcedureMeta.Deserializer.perform( + new ByteArrayInputStream( + procedureYaml.getBytes())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toMap(StoredProcedureMeta::getName, k -> k)); } } - @Override public @Nullable StoredProcedureMeta getStoredProcedure(String procedureName) { - return this.storedProcedureMetaMap.get(procedureName); + return getStoredProcedure(procedureName, true); + } + + /** + * + * @param procedureName + * @param update determine whether a remote update request should be sent to the admin service when the procedure is not found in the local cache + * @return + */ + private @Nullable StoredProcedureMeta getStoredProcedure(String procedureName, boolean update) { + StoredProcedureMeta cachedProcedure = this.storedProcedureMetaMap.get(procedureName); + if (cachedProcedure == null) { + if (update && this.metaUpdater != null) { + IrMeta meta = this.metaUpdater.onUpdate(); + if (meta != null && meta.getStoredProcedures() != null) { + return meta.getStoredProcedures().getStoredProcedure(procedureName, false); + } + } + } + return cachedProcedure; + } + + public void registerIrMetaUpdater(IrMetaUpdater updater) { + this.metaUpdater = updater; + } + + @Override + public String toString() { + return storedProcedureMetaMap.keySet().toString(); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java new file mode 100644 index 000000000000..76f07c678902 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java @@ -0,0 +1,96 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta.reader; + +import com.alibaba.graphscope.common.client.channel.ChannelFetcher; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; +import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; + +import org.yaml.snakeyaml.Yaml; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +// read ir meta from a remote http service +public class HttpIrMetaReader implements IrMetaReader { + private static final String CONTENT_TYPE = "Content-Type"; + private static final String APPLICATION_JSON = "application/json; utf-8"; + private final ChannelFetcher fetcher; + private final HttpClient httpClient; + + public HttpIrMetaReader(ChannelFetcher fetcher) { + this.fetcher = fetcher; + this.httpClient = HttpClient.newBuilder().build(); + } + + @Override + public IrMeta readMeta() throws IOException { + try { + List metaServiceHost = fetcher.fetch(); + Preconditions.checkArgument( + !metaServiceHost.isEmpty(), "can not get meta service host"); + URI getMetaUri = metaServiceHost.get(0).resolve("/v1/service/status"); + HttpRequest request = + HttpRequest.newBuilder() + .uri(getMetaUri) + .headers(CONTENT_TYPE, APPLICATION_JSON) + .GET() + .build(); + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + String res = response.body(); + Preconditions.checkArgument( + response.statusCode() == 200, + "get meta data fail, status code: %s, error message: %s", + response.statusCode(), + res); + String metaInYaml = convertMetaFromJsonToYaml(res); + return new IrMeta( + new IrGraphSchema( + new SchemaInputStream( + new ByteArrayInputStream( + metaInYaml.getBytes(StandardCharsets.UTF_8)), + FileFormatType.YAML)), + new GraphStoredProcedures( + new ByteArrayInputStream(metaInYaml.getBytes(StandardCharsets.UTF_8)))); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + private String convertMetaFromJsonToYaml(String metaInJson) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = mapper.readTree(metaInJson); + Map rootMap = mapper.convertValue(rootNode, Map.class); + Map metaMap = (Map) rootMap.get("graph"); + Yaml yaml = new Yaml(); + return yaml.dump(metaMap); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedures.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/IrMetaReader.java similarity index 65% rename from interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedures.java rename to interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/IrMetaReader.java index 7a4fc33d961c..bbe8b5e79294 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedures.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/IrMetaReader.java @@ -14,10 +14,15 @@ * limitations under the License. */ -package com.alibaba.graphscope.common.ir.meta.procedure; +package com.alibaba.graphscope.common.ir.meta.reader; -import org.checkerframework.checker.nullness.qual.Nullable; +import com.alibaba.graphscope.common.ir.meta.IrMeta; -public interface StoredProcedures { - @Nullable StoredProcedureMeta getStoredProcedure(String procedureName); +import java.io.IOException; + +/** + * {@code IrMetaReader} is used to read Ir Meta from a data source (can be a local file or remote web service). + */ +public interface IrMetaReader { + IrMeta readMeta() throws IOException; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java new file mode 100644 index 000000000000..508316c95281 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.ir.meta.reader; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.GraphConfig; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; +import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; +import com.alibaba.graphscope.common.utils.FileUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Objects; + +/** + * a local file system implementation of {@code IrMetaReader} + */ +public class LocalIrMetaReader implements IrMetaReader { + private static final Logger logger = LoggerFactory.getLogger(LocalIrMetaReader.class); + private final Configs configs; + + public LocalIrMetaReader(Configs configs) { + this.configs = configs; + } + + @Override + public IrMeta readMeta() throws IOException { + String metaPath = + Objects.requireNonNull( + GraphConfig.GRAPH_SCHEMA.get(configs), "ir meta path not exist"); + FileFormatType formatType = FileUtils.getFormatType(metaPath); + IrGraphSchema graphSchema = + new IrGraphSchema(new SchemaInputStream(new FileInputStream(metaPath), formatType)); + IrMeta irMeta = + (formatType == FileFormatType.YAML) + ? new IrMeta( + graphSchema, + new GraphStoredProcedures(new FileInputStream(metaPath))) + : new IrMeta(graphSchema); + return irMeta; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java deleted file mode 100644 index 615234bc9371..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalMetaDataReader.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.ir.meta.reader; - -import com.alibaba.graphscope.common.config.Configs; -import com.alibaba.graphscope.common.config.GraphConfig; -import com.alibaba.graphscope.common.utils.FileUtils; -import com.google.common.collect.ImmutableList; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; - -import java.io.ByteArrayInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -// a local file system implementation of MetaDataReader -public class LocalMetaDataReader implements MetaDataReader { - private static final Logger logger = LoggerFactory.getLogger(LocalMetaDataReader.class); - private final Configs configs; - - public LocalMetaDataReader(Configs configs) { - this.configs = configs; - } - - @Override - public List getStoredProcedures() throws IOException { - String storedProceduresYaml = GraphConfig.GRAPH_STORED_PROCEDURES_YAML.get(configs); - if (StringUtils.isEmpty(storedProceduresYaml)) { - return ImmutableList.of(); - } - Yaml yaml = new Yaml(); - Object raw = yaml.load(storedProceduresYaml); - if (!(raw instanceof List)) { - logger.error("stored procedures yaml format error"); - return ImmutableList.of(); - } - List procedureList = (List) raw; - return procedureList.stream() - .map( - k -> { - String procedureYaml = yaml.dump(k); - return new ByteArrayInputStream(procedureYaml.getBytes()); - }) - .collect(Collectors.toList()); - } - - @Override - public SchemaInputStream getGraphSchema() throws IOException { - String schemaPath = - Objects.requireNonNull( - GraphConfig.GRAPH_SCHEMA.get(configs), "schema path not exist"); - return new SchemaInputStream( - new FileInputStream(schemaPath), FileUtils.getFormatType(schemaPath)); - } -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/MetaDataReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/MetaDataReader.java deleted file mode 100644 index c53a25598595..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/MetaDataReader.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.ir.meta.reader; - -import java.io.InputStream; -import java.util.List; - -// MetaDataReader is used to read meta data from a data source (can be a local file or -// remote web service) -public interface MetaDataReader { - - // if enableProcedures is null, return all stored procedures - List getStoredProcedures() throws Exception; - - SchemaInputStream getGraphSchema() throws Exception; -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java index bdec124ad9fe..238b6b4d0ba5 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java @@ -16,13 +16,13 @@ package com.alibaba.graphscope.common.ir.meta.schema; -import com.alibaba.graphscope.common.ir.meta.reader.MetaDataReader; import com.alibaba.graphscope.common.ir.meta.reader.SchemaInputStream; import com.alibaba.graphscope.groot.common.exception.GraphElementNotFoundException; import com.alibaba.graphscope.groot.common.exception.GraphPropertyNotFoundException; import com.alibaba.graphscope.groot.common.schema.api.*; import com.alibaba.graphscope.groot.common.util.IrSchemaParser; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -35,9 +35,8 @@ public class IrGraphSchema implements GraphSchema { private final String schemeJson; private final boolean isColumnId; - public IrGraphSchema(MetaDataReader dataReader) throws Exception { + public IrGraphSchema(SchemaInputStream schemaInputStream) throws IOException { this.isColumnId = false; - SchemaInputStream schemaInputStream = dataReader.getGraphSchema(); String content = new String( schemaInputStream.getInputStream().readAllBytes(), StandardCharsets.UTF_8); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphIOProcessor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphIOProcessor.java index f630e3fbb13a..ec6c44153e51 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphIOProcessor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphIOProcessor.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.common.ir.planner; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.schema.CommonOptTable; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; import com.alibaba.graphscope.common.ir.planner.type.DataKey; @@ -38,7 +39,6 @@ import com.alibaba.graphscope.common.ir.tools.config.*; import com.alibaba.graphscope.common.ir.type.GraphLabelType; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.groot.common.schema.api.EdgeRelation; import com.alibaba.graphscope.groot.common.schema.api.GraphEdge; import com.alibaba.graphscope.groot.common.schema.api.GraphVertex; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java index a42c294c7c70..d9f08d30f458 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java @@ -19,6 +19,7 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.config.PegasusConfig; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.rel.*; import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate; import com.alibaba.graphscope.common.ir.rel.GraphLogicalProject; @@ -39,7 +40,6 @@ import com.alibaba.graphscope.common.jna.type.FfiData; import com.alibaba.graphscope.common.jna.type.FfiResult; import com.alibaba.graphscope.common.jna.type.ResultCode; -import com.alibaba.graphscope.common.store.IrMeta; import com.google.common.base.Preconditions; import com.sun.jna.Pointer; import com.sun.jna.ptr.IntByReference; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelProtoPhysicalBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelProtoPhysicalBuilder.java index e33739390874..b056aeaff89d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelProtoPhysicalBuilder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelProtoPhysicalBuilder.java @@ -17,13 +17,13 @@ package com.alibaba.graphscope.common.ir.runtime.proto; import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.schema.CommonOptTable; import com.alibaba.graphscope.common.ir.rel.CommonTableScan; import com.alibaba.graphscope.common.ir.rel.GraphShuttle; import com.alibaba.graphscope.common.ir.runtime.PhysicalBuilder; import com.alibaba.graphscope.common.ir.runtime.PhysicalPlan; import com.alibaba.graphscope.common.ir.tools.LogicalPlan; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.gaia.proto.GraphAlgebra; import com.alibaba.graphscope.gaia.proto.GraphAlgebraPhysical; import com.google.common.base.Preconditions; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 15eaa137eb72..15e847e6ddfc 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -18,8 +18,10 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; -import com.alibaba.graphscope.common.ir.meta.reader.LocalMetaDataReader; +import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader; import com.alibaba.graphscope.common.ir.meta.schema.GraphOptSchema; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; import com.alibaba.graphscope.common.ir.planner.GraphIOProcessor; @@ -30,8 +32,6 @@ import com.alibaba.graphscope.common.ir.runtime.ffi.FfiPhysicalBuilder; import com.alibaba.graphscope.common.ir.runtime.proto.GraphRelProtoPhysicalBuilder; import com.alibaba.graphscope.common.ir.type.GraphTypeFactoryImpl; -import com.alibaba.graphscope.common.store.ExperimentalMetaFetcher; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.cypher.antlr4.parser.CypherAntlr4Parser; import com.alibaba.graphscope.cypher.antlr4.visitor.LogicalPlanVisitor; import com.google.common.base.Preconditions; @@ -196,8 +196,7 @@ public static void main(String[] args) throws Exception { + " 'optional '"); } Configs configs = Configs.Factory.create(args[0]); - ExperimentalMetaFetcher metaFetcher = - new ExperimentalMetaFetcher(new LocalMetaDataReader(configs)); + StaticIrMetaFetcher metaFetcher = new StaticIrMetaFetcher(new LocalIrMetaReader(configs)); String query = FileUtils.readFileToString(new File(args[1]), StandardCharsets.UTF_8); GraphPlanner planner = new GraphPlanner( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/LogicalPlanFactory.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/LogicalPlanFactory.java index 276927dff5ea..069062046641 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/LogicalPlanFactory.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/LogicalPlanFactory.java @@ -16,7 +16,7 @@ package com.alibaba.graphscope.common.ir.tools; -import com.alibaba.graphscope.common.store.IrMeta; +import com.alibaba.graphscope.common.ir.meta.IrMeta; public interface LogicalPlanFactory { LogicalPlan create(GraphBuilder builder, IrMeta irMeta, String query); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/QueryCache.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/QueryCache.java index 1e40971112b8..7832667722df 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/QueryCache.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/QueryCache.java @@ -18,8 +18,8 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.runtime.PhysicalPlan; -import com.alibaba.graphscope.common.store.IrMeta; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/manager/IrMetaQueryCallback.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/manager/IrMetaQueryCallback.java index 98605ae1330f..0d644167cb42 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/manager/IrMetaQueryCallback.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/manager/IrMetaQueryCallback.java @@ -16,8 +16,8 @@ package com.alibaba.graphscope.common.manager; -import com.alibaba.graphscope.common.store.IrMeta; -import com.alibaba.graphscope.common.store.IrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; import java.util.Optional; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/ExperimentalMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/ExperimentalMetaFetcher.java deleted file mode 100644 index 263fdbbfe0f2..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/ExperimentalMetaFetcher.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.store; - -import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; -import com.alibaba.graphscope.common.ir.meta.reader.MetaDataReader; -import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; - -import java.util.Optional; - -public class ExperimentalMetaFetcher implements IrMetaFetcher { - private final IrMeta meta; - - public ExperimentalMetaFetcher(MetaDataReader dataReader) throws Exception { - this.meta = - new IrMeta(new IrGraphSchema(dataReader), new GraphStoredProcedures(dataReader)); - } - - @Override - public Optional fetch() { - return Optional.of(this.meta); - } -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMeta.java deleted file mode 100644 index 5e7ecd2e64d2..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMeta.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.store; - -import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedures; -import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; - -import org.checkerframework.checker.nullness.qual.Nullable; - -import java.io.IOException; -import java.util.Objects; - -public class IrMeta { - private final SnapshotId snapshotId; - private final IrGraphSchema schema; - private final @Nullable StoredProcedures storedProcedures; - - public IrMeta(IrGraphSchema schema) throws IOException { - this(SnapshotId.createEmpty(), schema); - } - - public IrMeta(IrGraphSchema schema, StoredProcedures storedProcedures) { - this(SnapshotId.createEmpty(), schema, storedProcedures); - } - - public IrMeta(SnapshotId snapshotId, IrGraphSchema schema) throws IOException { - this.snapshotId = Objects.requireNonNull(snapshotId); - this.schema = Objects.requireNonNull(schema); - this.storedProcedures = null; - } - - public IrMeta(SnapshotId snapshotId, IrGraphSchema schema, StoredProcedures storedProcedures) { - this.snapshotId = Objects.requireNonNull(snapshotId); - this.schema = Objects.requireNonNull(schema); - this.storedProcedures = Objects.requireNonNull(storedProcedures); - } - - public IrGraphSchema getSchema() { - return schema; - } - - public SnapshotId getSnapshotId() { - return snapshotId; - } - - public @Nullable StoredProcedures getStoredProcedures() { - return storedProcedures; - } -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMetaFetcher.java deleted file mode 100644 index 6017a4b030c5..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMetaFetcher.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.store; - -import java.util.Optional; - -public interface IrMetaFetcher { - Optional fetch(); -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/SnapshotId.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/SnapshotId.java deleted file mode 100644 index 095143d4f942..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/SnapshotId.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.store; - -public class SnapshotId { - public static final SnapshotId createEmpty() { - return new SnapshotId(false, -1); - } - - private final boolean acquired; - private final long id; - - public SnapshotId(boolean acquired, long id) { - this.acquired = acquired; - this.id = id; - } - - public boolean isAcquired() { - return acquired; - } - - public long getId() { - return id; - } -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/LogicalPlanVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/LogicalPlanVisitor.java index bfd51e6f475a..8472fd682812 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/LogicalPlanVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/LogicalPlanVisitor.java @@ -16,13 +16,13 @@ package com.alibaba.graphscope.cypher.antlr4.visitor; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; import com.alibaba.graphscope.common.ir.rel.graph.AbstractBindableTableScan; import com.alibaba.graphscope.common.ir.rel.graph.match.GraphLogicalMultiMatch; import com.alibaba.graphscope.common.ir.rel.graph.match.GraphLogicalSingleMatch; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.LogicalPlan; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.grammar.CypherGSBaseVisitor; import com.alibaba.graphscope.grammar.CypherGSParser; import com.google.common.collect.ImmutableList; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ProcedureCallVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ProcedureCallVisitor.java index 73573b270870..ffaf9e4f2221 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ProcedureCallVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ProcedureCallVisitor.java @@ -16,11 +16,11 @@ package com.alibaba.graphscope.cypher.antlr4.visitor; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; -import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedures; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.grammar.CypherGSBaseVisitor; import com.alibaba.graphscope.grammar.CypherGSParser; import com.google.common.base.Preconditions; @@ -79,7 +79,7 @@ public RexNode visitOC_ImplicitProcedureInvocation( public SqlOperator visitOC_ProcedureNameAsOperator(CypherGSParser.OC_ProcedureNameContext ctx) { String procedureName = ctx.getText(); StoredProcedureMeta meta = null; - StoredProcedures procedures = irMeta.getStoredProcedures(); + GraphStoredProcedures procedures = irMeta.getStoredProcedures(); Preconditions.checkArgument( procedures != null && (meta = procedures.getStoredProcedure(procedureName)) != null, "procedure %s not found", diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/executor/GraphQueryExecutor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/executor/GraphQueryExecutor.java index ff7e914a6791..0abe0b3cfc0f 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/executor/GraphQueryExecutor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/executor/GraphQueryExecutor.java @@ -21,11 +21,11 @@ import com.alibaba.graphscope.common.client.type.ExecutionResponseListener; import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.QueryTimeoutConfig; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.tools.GraphPlanner; import com.alibaba.graphscope.common.ir.tools.QueryCache; import com.alibaba.graphscope.common.ir.tools.QueryIdGenerator; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.gaia.proto.IrResult; import com.google.common.base.Preconditions; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java index 48f820a479cb..42e06e957137 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java @@ -22,11 +22,11 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.config.QueryTimeoutConfig; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.tools.GraphPlanner; import com.alibaba.graphscope.common.ir.tools.QueryCache; import com.alibaba.graphscope.common.ir.tools.QueryIdGenerator; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.gaia.proto.IrResult; import com.alibaba.graphscope.gremlin.integration.result.GraphProperties; import com.alibaba.graphscope.gremlin.integration.resultx.GremlinTestRecordParser; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java index 6e41705582e4..2f54bef0a07e 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java @@ -33,10 +33,10 @@ import com.alibaba.graphscope.common.config.PegasusConfig; import com.alibaba.graphscope.common.config.QueryTimeoutConfig; import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.tools.QueryCache; import com.alibaba.graphscope.common.ir.tools.QueryIdGenerator; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.gremlin.InterOpCollectionBuilder; import com.alibaba.graphscope.gremlin.Utils; import com.alibaba.graphscope.gremlin.plugin.MetricsCollector; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/LifeCycleSupplier.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/LifeCycleSupplier.java index 31541842a1b6..79691bc64cca 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/LifeCycleSupplier.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/LifeCycleSupplier.java @@ -20,9 +20,9 @@ import com.alibaba.graphscope.common.client.type.ExecutionRequest; import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.QueryTimeoutConfig; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.tools.GraphPlanner; import com.alibaba.graphscope.common.ir.tools.QueryCache; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.gaia.proto.IrResult; import com.alibaba.graphscope.gremlin.plugin.QueryStatusCallback; import com.alibaba.graphscope.gremlin.resultx.GremlinRecordParser; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/GremlinCalciteScriptEngineFactory.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/GremlinCalciteScriptEngineFactory.java index b9609c73f891..018e1495fbf0 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/GremlinCalciteScriptEngineFactory.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/GremlinCalciteScriptEngineFactory.java @@ -16,8 +16,8 @@ package com.alibaba.graphscope.gremlin.plugin.script; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.tools.QueryCache; -import com.alibaba.graphscope.common.store.IrMeta; import org.apache.tinkerpop.gremlin.jsr223.AbstractGremlinScriptEngineFactory; import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine; diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java index 8382c92d3f52..addf56aef907 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java @@ -16,9 +16,10 @@ package com.alibaba.graphscope.common.config; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; -import com.alibaba.graphscope.common.ir.meta.reader.LocalMetaDataReader; +import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; import org.junit.Assert; @@ -36,8 +37,8 @@ public void hiactor_config_test() throws Exception { public void procedure_config_test() throws Exception { YamlConfigs configs = new YamlConfigs("config/gs_interactive_hiactor.yaml", FileLoadType.RESOURCES); - GraphStoredProcedures procedures = - new GraphStoredProcedures(new LocalMetaDataReader(configs)); + IrMeta irMeta = new LocalIrMetaReader(configs).readMeta(); + GraphStoredProcedures procedures = irMeta.getStoredProcedures(); StoredProcedureMeta meta = procedures.getStoredProcedure("ldbc_ic2"); Assert.assertEquals( "StoredProcedureMeta{name='ldbc_ic2', returnType=RecordType(CHAR(1) name)," @@ -75,7 +76,8 @@ public void pegasus_config_test() throws Exception { public void schema_config_test() throws Exception { YamlConfigs configs = new YamlConfigs("config/gs_interactive_hiactor.yaml", FileLoadType.RESOURCES); - IrGraphSchema graphSchema = new IrGraphSchema(new LocalMetaDataReader(configs)); + IrMeta irMeta = new LocalIrMetaReader(configs).readMeta(); + IrGraphSchema graphSchema = irMeta.getSchema(); Assert.assertEquals( "DefaultGraphVertex{labelId=0, label=person," + " propertyList=[DefaultGraphProperty{id=0, name=id, dataType=LONG}," diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/QueryCacheTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/QueryCacheTest.java index de2be9cc48fc..7878c19aaeb6 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/QueryCacheTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/QueryCacheTest.java @@ -17,10 +17,10 @@ package com.alibaba.graphscope.common.ir; import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.GraphPlanner; import com.alibaba.graphscope.common.ir.tools.QueryCache; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.cypher.antlr4.parser.CypherAntlr4Parser; import com.alibaba.graphscope.cypher.antlr4.visitor.LogicalPlanVisitor; import com.google.common.collect.ImmutableMap; diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/Utils.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/Utils.java index 021a778e1ebb..0cb50e54328a 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/Utils.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/Utils.java @@ -19,7 +19,9 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.config.GraphConfig; -import com.alibaba.graphscope.common.ir.meta.reader.LocalMetaDataReader; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader; import com.alibaba.graphscope.common.ir.meta.schema.GraphOptSchema; import com.alibaba.graphscope.common.ir.planner.GraphHepPlanner; import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; @@ -27,9 +29,6 @@ import com.alibaba.graphscope.common.ir.tools.GraphBuilderFactory; import com.alibaba.graphscope.common.ir.tools.GraphRexBuilder; import com.alibaba.graphscope.common.ir.type.GraphTypeFactoryImpl; -import com.alibaba.graphscope.common.store.ExperimentalMetaFetcher; -import com.alibaba.graphscope.common.store.IrMeta; -import com.alibaba.graphscope.common.utils.FileUtils; import com.google.common.collect.ImmutableMap; import org.apache.calcite.plan.GraphOptCluster; @@ -41,10 +40,8 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.tools.RelBuilderFactory; -import org.yaml.snakeyaml.Yaml; import java.net.URL; -import java.util.Map; public class Utils { public static final Configs configs = @@ -94,22 +91,16 @@ public static final RelOptPlanner mockPlanner(RelRule.Config... rules) { return new GraphHepPlanner(hepBuilder.build()); } - public static IrMeta mockSchemaMeta(String schemaJson) { + public static IrMeta mockSchemaMeta(String schemaPath) { try { URL schemaResource = - Thread.currentThread().getContextClassLoader().getResource(schemaJson); - String yamlStr = FileUtils.readJsonFromResource("config/modern/graph.yaml"); - Yaml yaml = new Yaml(); - Map yamlMap = yaml.load(yamlStr); - String proceduresYaml = yaml.dump(yamlMap.get("stored_procedures")); + Thread.currentThread().getContextClassLoader().getResource(schemaPath); Configs configs = new Configs( ImmutableMap.of( GraphConfig.GRAPH_SCHEMA.getKey(), - schemaResource.toURI().getPath(), - GraphConfig.GRAPH_STORED_PROCEDURES_YAML.getKey(), - proceduresYaml)); - return new ExperimentalMetaFetcher(new LocalMetaDataReader(configs)).fetch().get(); + schemaResource.toURI().getPath())); + return new StaticIrMetaFetcher(new LocalIrMetaReader(configs)).fetch().get(); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/BITest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/BITest.java index 479d4514704f..40829cb84651 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/BITest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/BITest.java @@ -20,10 +20,10 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.ir.Utils; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.planner.GraphIOProcessor; import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; -import com.alibaba.graphscope.common.store.IrMeta; import com.google.common.collect.ImmutableMap; import org.apache.calcite.rel.RelNode; diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/CBOTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/CBOTest.java index 2b15b71988be..a6f2586072c5 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/CBOTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/CBOTest.java @@ -2,10 +2,10 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.ir.Utils; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.planner.GraphIOProcessor; import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; -import com.alibaba.graphscope.common.store.IrMeta; import com.google.common.collect.ImmutableMap; import org.apache.calcite.rel.RelNode; diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java index 9a0e9db8c622..c7f3aba234fd 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java @@ -2,10 +2,10 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.ir.Utils; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.planner.GraphIOProcessor; import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; -import com.alibaba.graphscope.common.store.IrMeta; import com.google.common.collect.ImmutableMap; import org.apache.calcite.rel.RelNode; diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/STPathTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/STPathTest.java index 91e98da2fc54..7543784403d6 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/STPathTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/STPathTest.java @@ -20,13 +20,13 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.ir.Utils; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.planner.GraphIOProcessor; import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; import com.alibaba.graphscope.common.ir.runtime.PhysicalBuilder; import com.alibaba.graphscope.common.ir.runtime.proto.GraphRelProtoPhysicalBuilder; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.LogicalPlan; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.common.utils.FileUtils; import com.google.common.collect.ImmutableMap; diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java index 5f50439315db..da6232246bec 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java @@ -153,7 +153,7 @@ public void logical_plan_3_test() throws Exception { public void logical_plan_4_test() throws Exception { LogicalPlan logicalPlan = com.alibaba.graphscope.cypher.antlr4.Utils.evalLogicalPlan( - "Call ldbc_ic2(10l, 20120112l)"); + "Call ldbc_ic2(10l, 20120112l)", "config/modern/graph.yaml"); try (PhysicalBuilder ffiBuilder = new ProcedurePhysicalBuilder(logicalPlan)) { PhysicalPlan plan = ffiBuilder.build(); Assert.assertEquals( diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/GraphRelToProtoTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/GraphRelToProtoTest.java index 3fe7ce8e4e40..00b88df2231c 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/GraphRelToProtoTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/GraphRelToProtoTest.java @@ -18,6 +18,7 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.ir.Utils; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.planner.GraphIOProcessor; import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; import com.alibaba.graphscope.common.ir.planner.rules.DegreeFusionRule; @@ -32,7 +33,6 @@ import com.alibaba.graphscope.common.ir.tools.config.LabelConfig; import com.alibaba.graphscope.common.ir.tools.config.PathExpandConfig; import com.alibaba.graphscope.common.ir.tools.config.SourceConfig; -import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.common.utils.FileUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/CallProcedureTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/CallProcedureTest.java index 2d6bdd097b8a..21d58c06db55 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/CallProcedureTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/CallProcedureTest.java @@ -23,9 +23,11 @@ import org.junit.Test; public class CallProcedureTest { + @Test public void procedure_1_test() { - LogicalPlan logicalPlan = Utils.evalLogicalPlan("Call ldbc_ic2(10l, 20120112l)"); + LogicalPlan logicalPlan = + Utils.evalLogicalPlan("Call ldbc_ic2(10l, 20120112l)", "config/modern/graph.yaml"); Assert.assertEquals("ldbc_ic2(10:BIGINT, 20120112:BIGINT)", logicalPlan.explain().trim()); Assert.assertEquals( "RecordType(CHAR(1) name)", logicalPlan.getProcedureCall().getType().toString()); @@ -35,7 +37,7 @@ public void procedure_1_test() { @Test public void procedure_2_test() { try { - Utils.evalLogicalPlan("Call ldbc_ic2(10, 20120112l)"); + Utils.evalLogicalPlan("Call ldbc_ic2(10, 20120112l)", "config/modern/graph.yaml"); } catch (CalciteException e) { Assert.assertEquals( "Cannot apply ldbc_ic2 to arguments of type 'ldbc_ic2(, )'." diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/Utils.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/Utils.java index 66458285cce7..01f5d916d97a 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/Utils.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/Utils.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.cypher.antlr4; +import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.LogicalPlan; import com.alibaba.graphscope.cypher.antlr4.parser.CypherAntlr4Parser; @@ -39,4 +40,12 @@ public static LogicalPlan evalLogicalPlan(String query) { graphBuilder, com.alibaba.graphscope.common.ir.Utils.schemaMeta); return logicalPlanVisitor.visit(new CypherAntlr4Parser().parse(query)); } + + public static LogicalPlan evalLogicalPlan(String query, String schemaPath) { + GraphBuilder graphBuilder = + com.alibaba.graphscope.common.ir.Utils.mockGraphBuilder(schemaPath); + IrMeta irMeta = com.alibaba.graphscope.common.ir.Utils.mockSchemaMeta(schemaPath); + LogicalPlanVisitor logicalPlanVisitor = new LogicalPlanVisitor(graphBuilder, irMeta); + return logicalPlanVisitor.visit(new CypherAntlr4Parser().parse(query)); + } } diff --git a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/Frontend.java b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/Frontend.java index f3d6e67be719..5cebec0d8f88 100644 --- a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/Frontend.java +++ b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/Frontend.java @@ -20,7 +20,7 @@ import com.alibaba.graphscope.common.client.channel.ChannelFetcher; import com.alibaba.graphscope.common.client.channel.HostsRpcChannelFetcher; import com.alibaba.graphscope.common.config.Configs; -import com.alibaba.graphscope.common.config.GraphConfig; +import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; import com.alibaba.graphscope.gremlin.integration.result.TestGraphFactory; @@ -45,9 +45,8 @@ public Frontend(Configs configs) { public void start() throws Exception { ChannelFetcher channelFetcher = new HostsRpcChannelFetcher(configs); - String vineyardSchemaPath = GraphConfig.GRAPH_SCHEMA.get(configs); IrMetaQueryCallback queryCallback = - new IrMetaQueryCallback(new VineyardMetaFetcher(vineyardSchemaPath)); + new IrMetaQueryCallback(new StaticIrMetaFetcher(new VineyardIrMetaReader(configs))); this.graphServer = new GraphServer( this.configs, channelFetcher, queryCallback, TestGraphFactory.VINEYARD); diff --git a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/JsonFileSchemaFetcher.java b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/JsonFileSchemaFetcher.java deleted file mode 100644 index d1bc0e9e4a55..000000000000 --- a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/JsonFileSchemaFetcher.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.graphscope.frontend; - -import com.alibaba.graphscope.groot.common.schema.api.GraphSchema; -import com.alibaba.graphscope.groot.common.schema.api.SchemaFetcher; -import com.alibaba.graphscope.groot.common.schema.impl.DefaultGraphSchema; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Map; - -public class JsonFileSchemaFetcher implements SchemaFetcher { - private long snapshot; - private GraphSchema graphSchema; - - public JsonFileSchemaFetcher(String filePath) { - try { - String schemaString = new String(Files.readAllBytes(Paths.get(filePath))); - this.graphSchema = DefaultGraphSchema.buildSchemaFromJson(schemaString); - this.snapshot = Integer.MAX_VALUE - 1; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public Map getSchemaSnapshotPair() { - return Map.of(snapshot, graphSchema); - } - - @Override - public int getPartitionNum() { - return -1; - } - - @Override - public int getVersion() { - return -1; - } -} diff --git a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java new file mode 100644 index 000000000000..5a967b9abb88 --- /dev/null +++ b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.frontend; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.GraphConfig; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.reader.IrMetaReader; +import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; +import com.alibaba.graphscope.groot.common.schema.api.GraphSchema; +import com.alibaba.graphscope.groot.common.schema.impl.DefaultGraphSchema; + +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class VineyardIrMetaReader implements IrMetaReader { + private final Configs configs; + + public VineyardIrMetaReader(Configs configs) { + this.configs = configs; + } + + @Override + public IrMeta readMeta() throws IOException { + String schemaString = + FileUtils.readFileToString( + new File(GraphConfig.GRAPH_SCHEMA.get(configs)), StandardCharsets.UTF_8); + GraphSchema graphSchema = DefaultGraphSchema.buildSchemaFromJson(schemaString); + return new IrMeta(new IrGraphSchema(graphSchema, true)); + } +} diff --git a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardMetaFetcher.java b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardMetaFetcher.java deleted file mode 100644 index 7d5162b4c257..000000000000 --- a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardMetaFetcher.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.frontend; - -import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; -import com.alibaba.graphscope.common.store.IrMeta; -import com.alibaba.graphscope.common.store.IrMetaFetcher; -import com.alibaba.graphscope.groot.common.schema.api.GraphSchema; -import com.alibaba.graphscope.groot.common.util.IrSchemaParser; - -import java.io.IOException; -import java.util.Optional; - -public class VineyardMetaFetcher implements IrMetaFetcher { - private final IrMeta irMeta; - - public VineyardMetaFetcher(String schemaPath) { - IrSchemaParser parser = IrSchemaParser.getInstance(); - JsonFileSchemaFetcher fetcher = new JsonFileSchemaFetcher(schemaPath); - GraphSchema graphSchema = fetcher.getSchemaSnapshotPair().values().iterator().next(); - try { - this.irMeta = new IrMeta(new IrGraphSchema(graphSchema, true)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public Optional fetch() { - return Optional.of(irMeta); - } -} diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java index 9f467a4c5a9d..0330544d8c39 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java @@ -16,9 +16,9 @@ package com.alibaba.graphscope.groot.servers.ir; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; -import com.alibaba.graphscope.common.store.IrMeta; -import com.alibaba.graphscope.common.store.IrMetaFetcher; import com.alibaba.graphscope.groot.frontend.SnapshotUpdateClient; import com.alibaba.graphscope.groot.rpc.RoleClients; import com.google.common.util.concurrent.ThreadFactoryBuilder; diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootMetaFetcher.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootIrMetaReader.java similarity index 56% rename from interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootMetaFetcher.java rename to interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootIrMetaReader.java index 6a77d0f824da..ef07f5f841c0 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootMetaFetcher.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootIrMetaReader.java @@ -16,41 +16,32 @@ package com.alibaba.graphscope.groot.servers.ir; +import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.SnapshotId; +import com.alibaba.graphscope.common.ir.meta.reader.IrMetaReader; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; -import com.alibaba.graphscope.common.store.IrMeta; -import com.alibaba.graphscope.common.store.IrMetaFetcher; -import com.alibaba.graphscope.common.store.SnapshotId; import com.alibaba.graphscope.groot.common.schema.api.GraphSchema; import com.alibaba.graphscope.groot.common.schema.api.SchemaFetcher; import java.io.IOException; import java.util.*; -public class GrootMetaFetcher implements IrMetaFetcher { +public class GrootIrMetaReader implements IrMetaReader { private final SchemaFetcher schemaFetcher; - public GrootMetaFetcher(SchemaFetcher schemaFetcher) { + public GrootIrMetaReader(SchemaFetcher schemaFetcher) { this.schemaFetcher = schemaFetcher; } @Override - public Optional fetch() { + public IrMeta readMeta() throws IOException { Map pair = this.schemaFetcher.getSchemaSnapshotPair(); - - if (!pair.isEmpty()) { - Map.Entry entry = pair.entrySet().iterator().next(); - Long snapshotId = entry.getKey(); - GraphSchema schema = entry.getValue(); - - try { - return Optional.of( - new IrMeta( - new SnapshotId(true, snapshotId), new IrGraphSchema(schema, true))); - } catch (IOException e) { - throw new RuntimeException(e); - } - } else { - return Optional.empty(); + if (pair.isEmpty()) { + return null; } + Map.Entry entry = pair.entrySet().iterator().next(); + Long snapshotId = entry.getKey(); + GraphSchema schema = entry.getValue(); + return new IrMeta(new SnapshotId(true, snapshotId), new IrGraphSchema(schema, true)); } } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java index fc39e1d5ff69..9b0f8c73d500 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java @@ -21,7 +21,8 @@ import com.alibaba.graphscope.common.config.AuthConfig; import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.config.PegasusConfig; -import com.alibaba.graphscope.common.store.IrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.fetcher.DynamicIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; import com.alibaba.graphscope.gremlin.integration.result.TestGraphFactory; import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.CommonConfig; @@ -56,7 +57,8 @@ public AbstractService makeGraphService( new RpcChannelManagerFetcher(channelManager, executorCount, RoleType.GAIA_RPC); com.alibaba.graphscope.common.config.Configs irConfigs = getConfigs(); logger.info("IR configs: {}", irConfigs); - IrMetaFetcher irMetaFetcher = new GrootMetaFetcher(schemaFetcher); + IrMetaFetcher irMetaFetcher = + new DynamicIrMetaFetcher(new GrootIrMetaReader(schemaFetcher), irConfigs); RoleClients updateCommitter = new RoleClients<>(channelManager, RoleType.COORDINATOR, SnapshotUpdateClient::new); int frontendId = CommonConfig.NODE_IDX.get(configs); @@ -121,6 +123,8 @@ private com.alibaba.graphscope.common.config.Configs getConfigs() { addToConfigMapIfExist(FrontendConfig.FRONTEND_SERVER_NUM.getKey(), configMap); // add frontend qps limit addToConfigMapIfExist(FrontendConfig.QUERY_PER_SECOND_LIMIT.getKey(), configMap); + // add ir meta fetch interval + addToConfigMapIfExist(FrontendConfig.IR_META_FETCH_INTERVAL_MS.getKey(), configMap); return new com.alibaba.graphscope.common.config.Configs(configMap); } From 558255360a37317a37b295f3d7c7d555f83c8c24 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 29 May 2024 20:55:00 +0800 Subject: [PATCH 02/25] remove useless files --- interactive_engine/compiler/graph.yaml | 93 --------------------- interactive_engine/compiler/graph_test.json | 1 - 2 files changed, 94 deletions(-) delete mode 100644 interactive_engine/compiler/graph.yaml delete mode 100644 interactive_engine/compiler/graph_test.json diff --git a/interactive_engine/compiler/graph.yaml b/interactive_engine/compiler/graph.yaml deleted file mode 100644 index 85a94f9077c1..000000000000 --- a/interactive_engine/compiler/graph.yaml +++ /dev/null @@ -1,93 +0,0 @@ -name: modern # then must have a modern dir under ${data} directory -store_type: mutable_csr # v6d, groot, gart -stored_procedures: - - name: ldbc_ic2 - description: "A stored procedures for the ldbc complex interactive workload 2" - type: x_cypher - params: - - name: personId2 - type: - primitive_type: DT_SIGNED_INT64 - - name: maxDate - type: - primitive_type: DT_SIGNED_INT64 - returns: - - name: name - type: - string: - long_text: - query: "MATCH(n: PERSON ${personId2}) WHERE n.creationDate < ${maxDate} RETURN n.firstName AS name LIMIT 10;" - library: libquery_ic2.so - encoding: string -schema: - vertex_types: - - type_name: person - type_id: 0 - x_csr_params: - max_vertex_num: 100 - properties: - - property_id: 0 - property_name: id - property_type: - primitive_type: DT_SIGNED_INT64 - - property_id: 1 - property_name: name - property_type: - string: - long_text: - - property_id: 2 - property_name: age - property_type: - primitive_type: DT_SIGNED_INT32 - primary_keys: - - id - - type_name: software - type_id: 1 - x_csr_params: - max_vertex_num: 100 - properties: - - property_id: 0 - property_name: id - property_type: - primitive_type: DT_SIGNED_INT64 - x_csr_params: - - property_id: 1 - property_name: name - property_type: - string: - long_text: - - property_id: 2 - property_name: lang - property_type: - string: - long_text: - - property_id: 3 - property_name: creationDate - property_type: - temporal: - date32: - primary_keys: - - id - edge_types: - - type_name: knows - type_id: 2 - vertex_type_pair_relations: - - source_vertex: person - destination_vertex: person - relation: MANY_TO_MANY - properties: - - property_id: 0 - property_name: weight - property_type: - primitive_type: DT_DOUBLE - - type_name: created - type_id: 3 - vertex_type_pair_relations: - - source_vertex: person - destination_vertex: software - relation: MANY_TO_MANY - properties: - - property_id: 0 - property_name: weight - property_type: - primitive_type: DT_DOUBLE diff --git a/interactive_engine/compiler/graph_test.json b/interactive_engine/compiler/graph_test.json deleted file mode 100644 index 606d2dac7c11..000000000000 --- a/interactive_engine/compiler/graph_test.json +++ /dev/null @@ -1 +0,0 @@ -{"bolt_port":7687,"graph":{"creation_time":1716969975953,"data_update_time":1716969975953,"description":"","id":"1","name":"ldbc_snb","schema":{"edge_types":[{"type_id":0,"type_name":"HASCREATOR","vertex_type_pair_relations":[{"destination_vertex":"PERSON","relation":"MANY_TO_ONE","source_vertex":"COMMENT"},{"destination_vertex":"PERSON","relation":"MANY_TO_ONE","source_vertex":"POST"}]},{"type_id":1,"type_name":"HASTAG","vertex_type_pair_relations":[{"destination_vertex":"TAG","relation":"MANY_TO_MANY","source_vertex":"POST"},{"destination_vertex":"TAG","relation":"MANY_TO_MANY","source_vertex":"FORUM"},{"destination_vertex":"TAG","relation":"MANY_TO_MANY","source_vertex":"COMMENT"}]},{"type_id":2,"type_name":"REPLYOF","vertex_type_pair_relations":[{"destination_vertex":"COMMENT","relation":"MANY_TO_MANY","source_vertex":"COMMENT"},{"destination_vertex":"POST","relation":"MANY_TO_ONE","source_vertex":"COMMENT"}]},{"type_id":3,"type_name":"CONTAINEROF","vertex_type_pair_relations":[{"destination_vertex":"POST","relation":"ONE_TO_MANY","source_vertex":"FORUM"}]},{"properties":[{"property_id":0,"property_name":"joinDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}}],"type_id":4,"type_name":"HASMEMBER","vertex_type_pair_relations":[{"destination_vertex":"PERSON","relation":"MANY_TO_MANY","source_vertex":"FORUM"}]},{"type_id":5,"type_name":"HASMODERATOR","vertex_type_pair_relations":[{"destination_vertex":"PERSON","relation":"MANY_TO_ONE","source_vertex":"FORUM"}]},{"type_id":6,"type_name":"HASINTEREST","vertex_type_pair_relations":[{"destination_vertex":"TAG","relation":"MANY_TO_MANY","source_vertex":"PERSON"}]},{"type_id":7,"type_name":"ISLOCATEDIN","vertex_type_pair_relations":[{"destination_vertex":"PLACE","relation":"MANY_TO_ONE","source_vertex":"COMMENT"},{"destination_vertex":"PLACE","relation":"MANY_TO_ONE","source_vertex":"PERSON"},{"destination_vertex":"PLACE","relation":"MANY_TO_ONE","source_vertex":"POST"},{"destination_vertex":"PLACE","relation":"MANY_TO_ONE","source_vertex":"ORGANISATION"}]},{"properties":[{"property_id":0,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}}],"type_id":8,"type_name":"KNOWS","vertex_type_pair_relations":[{"destination_vertex":"PERSON","relation":"MANY_TO_MANY","source_vertex":"PERSON"}]},{"properties":[{"property_id":0,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}}],"type_id":9,"type_name":"LIKES","vertex_type_pair_relations":[{"destination_vertex":"COMMENT","relation":"MANY_TO_MANY","source_vertex":"PERSON"},{"destination_vertex":"POST","relation":"MANY_TO_MANY","source_vertex":"PERSON"}]},{"properties":[{"property_id":0,"property_name":"workFrom","property_type":{"primitive_type":"DT_SIGNED_INT32"}}],"type_id":10,"type_name":"WORKAT","vertex_type_pair_relations":[{"destination_vertex":"ORGANISATION","relation":"MANY_TO_MANY","source_vertex":"PERSON"}]},{"type_id":11,"type_name":"ISPARTOF","vertex_type_pair_relations":[{"destination_vertex":"PLACE","relation":"MANY_TO_MANY","source_vertex":"PLACE"}]},{"type_id":12,"type_name":"HASTYPE","vertex_type_pair_relations":[{"destination_vertex":"TAGCLASS","relation":"MANY_TO_ONE","source_vertex":"TAG"}]},{"type_id":13,"type_name":"ISSUBCLASSOF","vertex_type_pair_relations":[{"destination_vertex":"TAGCLASS","relation":"MANY_TO_MANY","source_vertex":"TAGCLASS"}]},{"properties":[{"property_id":0,"property_name":"classYear","property_type":{"primitive_type":"DT_SIGNED_INT32"}}],"type_id":14,"type_name":"STUDYAT","vertex_type_pair_relations":[{"destination_vertex":"ORGANISATION","relation":"MANY_TO_MANY","source_vertex":"PERSON"}]}],"vertex_types":[{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"name","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"url","property_type":{"string":{"long_text":null}}},{"property_id":3,"property_name":"type","property_type":{"string":{"long_text":null}}}],"type_id":0,"type_name":"PLACE","x_csr_params":{"max_vertex_num":1460}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"firstName","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"lastName","property_type":{"string":{"long_text":null}}},{"property_id":3,"property_name":"gender","property_type":{"string":{"long_text":null}}},{"property_id":4,"property_name":"birthday","property_type":{"temporal":{"timestamp":null}}},{"property_id":5,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":6,"property_name":"locationIP","property_type":{"string":{"long_text":null}}},{"property_id":7,"property_name":"browserUsed","property_type":{"string":{"long_text":null}}},{"property_id":8,"property_name":"language","property_type":{"string":{"long_text":null}}},{"property_id":9,"property_name":"email","property_type":{"string":{"long_text":null}}}],"type_id":1,"type_name":"PERSON","x_csr_params":{"max_vertex_num":1254000}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":2,"property_name":"locationIP","property_type":{"string":{"long_text":null}}},{"property_id":3,"property_name":"browserUsed","property_type":{"string":{"long_text":null}}},{"property_id":4,"property_name":"content","property_type":{"string":{"long_text":null}}},{"property_id":5,"property_name":"length","property_type":{"primitive_type":"DT_SIGNED_INT32"}}],"type_id":2,"type_name":"COMMENT","x_csr_params":{"max_vertex_num":762949871}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"imageFile","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":3,"property_name":"locationIP","property_type":{"string":{"long_text":null}}},{"property_id":4,"property_name":"browserUsed","property_type":{"string":{"long_text":null}}},{"property_id":5,"property_name":"language","property_type":{"string":{"long_text":null}}},{"property_id":6,"property_name":"content","property_type":{"string":{"long_text":null}}},{"property_id":7,"property_name":"length","property_type":{"primitive_type":"DT_SIGNED_INT32"}}],"type_id":3,"type_name":"POST","x_csr_params":{"max_vertex_num":187893047}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"title","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"creationDate","property_type":{"primitive_type":"DT_SIGNED_INT64"}}],"type_id":4,"type_name":"FORUM","x_csr_params":{"max_vertex_num":12549398}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"type","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"name","property_type":{"string":{"long_text":null}}},{"property_id":3,"property_name":"url","property_type":{"string":{"long_text":null}}}],"type_id":5,"type_name":"ORGANISATION","x_csr_params":{"max_vertex_num":7955}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"name","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"url","property_type":{"string":{"long_text":null}}}],"type_id":6,"type_name":"TAGCLASS","x_csr_params":{"max_vertex_num":71}},{"primary_keys":["id"],"properties":[{"property_id":0,"property_name":"id","property_type":{"primitive_type":"DT_SIGNED_INT64"}},{"property_id":1,"property_name":"name","property_type":{"string":{"long_text":null}}},{"property_id":2,"property_name":"url","property_type":{"string":{"long_text":null}}}],"type_id":7,"type_name":"TAG","x_csr_params":{"max_vertex_num":16080}}]},"store_type":"mutable_csr","stored_procedures":[]},"gremlin_port":8182,"hqps_port":10000,"start_time":1716974357943,"status":"Stopped"} \ No newline at end of file From d35c5bb4fccdb832c5ca2a19e6c57aaa9ae104eb Mon Sep 17 00:00:00 2001 From: shirly121 Date: Wed, 29 May 2024 21:17:55 +0800 Subject: [PATCH 03/25] minor fix --- .../common/client/channel/MetaServiceChannelFetcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java index 531fb1d2906b..e7bfc5e71637 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java @@ -28,7 +28,7 @@ // read meta service host from config and return it as URI format public class MetaServiceChannelFetcher implements ChannelFetcher { private static final String schema = "http"; - private Configs graphConfig; + private final Configs graphConfig; public MetaServiceChannelFetcher(Configs graphConfig) { this.graphConfig = graphConfig; From 3513782b9d4e1db90e3e020543064eb3d13f496b Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 30 May 2024 21:32:19 +0800 Subject: [PATCH 04/25] minor fix --- .../java/com/alibaba/graphscope/common/ir/meta/IrMeta.java | 2 +- .../com/alibaba/graphscope/common/ir/meta/IrMetaStats.java | 3 +++ .../com/alibaba/graphscope/common/ir/meta/IrMetaTracker.java | 2 +- .../com/alibaba/graphscope/common/ir/planner/GlogueHolder.java | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java index d4de5d34b471..b9566922d558 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java @@ -26,7 +26,7 @@ import java.util.Objects; /** - * This class describes all the metadata information required by IR, including Schema, Procedures, and Statistics. + * This class describes basic metadata information required by IR, including Schema, Procedures. * They are versioned using the same snapshot id. */ public class IrMeta { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaStats.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaStats.java index d3e64fcd0bb4..77af8908b7fd 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaStats.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaStats.java @@ -24,6 +24,9 @@ import org.checkerframework.checker.nullness.qual.Nullable; +/** + * This class describes all metadata information required by IR, including Schema, Procedures and Statistics. + */ public class IrMetaStats extends IrMeta { private final @Nullable GraphStatistics statistics; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaTracker.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaTracker.java index 9f15c512cb7d..f5dbd7b4e82a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaTracker.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaTracker.java @@ -18,7 +18,7 @@ package com.alibaba.graphscope.common.ir.meta; -// This interface triggers the update of Meta Data +// This interface defines the callback to be invoked when IrMeta changes. public interface IrMetaTracker { void onChanged(IrMeta meta); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GlogueHolder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GlogueHolder.java index 58de10d1288c..e083133da51f 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GlogueHolder.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GlogueHolder.java @@ -31,6 +31,8 @@ import java.util.concurrent.atomic.AtomicReference; +// When IrMeta is updated, Glogue will also be triggered to update. This class defines the specific +// operations for updating Glogue. public class GlogueHolder implements IrMetaTracker { private final PlannerConfig config; private final AtomicReference glogueRef; From 6c3a32a49527c25c857f612b06d1de166fd46f91 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 31 May 2024 12:09:14 +0800 Subject: [PATCH 05/25] refine reading statistics interface --- .../graphscope/common/ir/meta/GraphId.java | 39 +++++++++++++++++++ .../graphscope/common/ir/meta/IrMeta.java | 15 +++++-- .../common/ir/meta/IrMetaStats.java | 11 +++++- .../ir/meta/fetcher/DynamicIrMetaFetcher.java | 2 +- .../ir/meta/fetcher/StaticIrMetaFetcher.java | 6 +-- .../ir/meta/reader/HttpIrMetaReader.java | 20 +++++++--- .../common/ir/meta/reader/IrMetaReader.java | 4 +- .../ir/meta/reader/LocalIrMetaReader.java | 3 +- .../frontend/VineyardIrMetaReader.java | 3 +- .../groot/servers/ir/GrootIrMetaReader.java | 3 +- 10 files changed, 89 insertions(+), 17 deletions(-) create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/GraphId.java diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/GraphId.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/GraphId.java new file mode 100644 index 000000000000..bda93af32ab9 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/GraphId.java @@ -0,0 +1,39 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.meta; + +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * This class is used to uniquely identify a graph instance in scenarios that support multiple graphs. + * @param + */ +public class GraphId { + public static final GraphId DEFAULT = new GraphId(null); + + private final @Nullable T id; + + public GraphId(T id) { + this.id = id; + } + + public @Nullable T getId() { + return id; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java index b9566922d558..6035e96490cb 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMeta.java @@ -30,6 +30,7 @@ * They are versioned using the same snapshot id. */ public class IrMeta { + protected final GraphId graphId; protected final SnapshotId snapshotId; protected final IrGraphSchema schema; protected final @Nullable GraphStoredProcedures storedProcedures; @@ -39,15 +40,19 @@ public IrMeta(IrGraphSchema schema) { } public IrMeta(IrGraphSchema schema, GraphStoredProcedures storedProcedures) { - this(SnapshotId.createEmpty(), schema, storedProcedures); + this(GraphId.DEFAULT, SnapshotId.createEmpty(), schema, storedProcedures); } public IrMeta(SnapshotId snapshotId, IrGraphSchema schema) { - this(snapshotId, schema, null); + this(GraphId.DEFAULT, snapshotId, schema, null); } public IrMeta( - SnapshotId snapshotId, IrGraphSchema schema, GraphStoredProcedures storedProcedures) { + GraphId graphId, + SnapshotId snapshotId, + IrGraphSchema schema, + GraphStoredProcedures storedProcedures) { + this.graphId = graphId; this.snapshotId = Objects.requireNonNull(snapshotId); this.schema = Objects.requireNonNull(schema); this.storedProcedures = storedProcedures; @@ -64,4 +69,8 @@ public SnapshotId getSnapshotId() { public @Nullable GraphStoredProcedures getStoredProcedures() { return storedProcedures; } + + public GraphId getGraphId() { + return graphId; + } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaStats.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaStats.java index 77af8908b7fd..b8a0b91f8090 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaStats.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/IrMetaStats.java @@ -35,7 +35,16 @@ public IrMetaStats( IrGraphSchema schema, GraphStoredProcedures storedProcedures, GraphStatistics statistics) { - super(snapshotId, schema, storedProcedures); + this(GraphId.DEFAULT, snapshotId, schema, storedProcedures, statistics); + } + + public IrMetaStats( + GraphId graphId, + SnapshotId snapshotId, + IrGraphSchema schema, + GraphStoredProcedures storedProcedures, + GraphStatistics statistics) { + super(graphId, snapshotId, schema, storedProcedures); this.statistics = statistics; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java index 6b3961d4774d..051f6ed8e17c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java @@ -83,7 +83,7 @@ private synchronized void syncMeta() { private synchronized void syncStats() { try { if (this.currentState != null) { - GraphStatistics stats = this.reader.readStats(); + GraphStatistics stats = this.reader.readStats(this.currentState.getGraphId()); if (stats != null) { this.currentState = new IrMetaStats( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java index 11d2ddc1b9c6..b48cc07d81a1 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java @@ -38,7 +38,7 @@ public class StaticIrMetaFetcher extends IrMetaFetcher { public StaticIrMetaFetcher(IrMetaReader dataReader, IrMetaTracker tracker) throws IOException { super(dataReader, tracker); IrMeta meta = this.reader.readMeta(); - GraphStatistics stats = fetchStats(); + GraphStatistics stats = (meta == null) ? null : fetchStats(meta); this.metaStats = new IrMetaStats( meta.getSnapshotId(), meta.getSchema(), meta.getStoredProcedures(), stats); @@ -47,9 +47,9 @@ public StaticIrMetaFetcher(IrMetaReader dataReader, IrMetaTracker tracker) throw } } - private @Nullable GraphStatistics fetchStats() { + private @Nullable GraphStatistics fetchStats(IrMeta meta) { try { - return this.reader.readStats(); + return this.reader.readStats(meta.getGraphId()); } catch (Exception e) { logger.warn("failed to read graph statistics, error is {}", e); return null; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java index 9ccd2e24ef47..97fad96e854c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java @@ -19,7 +19,9 @@ package com.alibaba.graphscope.common.ir.meta.reader; import com.alibaba.graphscope.common.client.channel.ChannelFetcher; +import com.alibaba.graphscope.common.ir.meta.GraphId; import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.SnapshotId; import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphStatistics; @@ -27,6 +29,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; +import org.javatuples.Pair; import org.yaml.snakeyaml.Yaml; import java.io.ByteArrayInputStream; @@ -61,8 +64,11 @@ public IrMeta readMeta() throws IOException { "read meta data fail, status code: %s, error message: %s", response.statusCode(), res); - String metaInYaml = convertMetaFromJsonToYaml(res); + Pair metaPair = convertMetaFromJsonToYaml(res); + String metaInYaml = metaPair.getValue1(); return new IrMeta( + metaPair.getValue0(), + SnapshotId.createEmpty(), // todo: return snapshot id from http service new IrGraphSchema( new SchemaInputStream( new ByteArrayInputStream( @@ -77,9 +83,12 @@ public IrMeta readMeta() throws IOException { } @Override - public IrGraphStatistics readStats() throws IOException { + public IrGraphStatistics readStats(GraphId graphId) throws IOException { try { - HttpResponse response = sendRequest("/v1/graph/current/statistics"); + Preconditions.checkArgument( + graphId.getId() != null, "graph id should not be null in http meta reader"); + HttpResponse response = + sendRequest(String.format("/v1/graph/%s/statistics", graphId.getId())); String res = response.body(); Preconditions.checkArgument( response.statusCode() == 200, @@ -107,12 +116,13 @@ private HttpResponse sendRequest(String requestUri) return httpClient.send(request, HttpResponse.BodyHandlers.ofString()); } - private String convertMetaFromJsonToYaml(String metaInJson) throws IOException { + private Pair convertMetaFromJsonToYaml(String metaInJson) throws IOException { ObjectMapper mapper = new ObjectMapper(); JsonNode rootNode = mapper.readTree(metaInJson); Map rootMap = mapper.convertValue(rootNode, Map.class); Map metaMap = (Map) rootMap.get("graph"); + GraphId graphId = new GraphId(metaMap.get("id")); Yaml yaml = new Yaml(); - return yaml.dump(metaMap); + return Pair.with(graphId, yaml.dump(metaMap)); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/IrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/IrMetaReader.java index dc7507f1f273..62e4142684b3 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/IrMetaReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/IrMetaReader.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.common.ir.meta.reader; +import com.alibaba.graphscope.common.ir.meta.GraphId; import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.groot.common.schema.api.GraphStatistics; @@ -27,5 +28,6 @@ public interface IrMetaReader { IrMeta readMeta() throws IOException; - GraphStatistics readStats() throws IOException; + // get statistics from a graph referenced by graphId + GraphStatistics readStats(GraphId graphId) throws IOException; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java index 82277188df6e..34e66cb8f01a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java @@ -18,6 +18,7 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.GraphConfig; +import com.alibaba.graphscope.common.ir.meta.GraphId; import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.procedure.GraphStoredProcedures; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; @@ -60,7 +61,7 @@ public IrMeta readMeta() throws IOException { } @Override - public IrGraphStatistics readStats() throws IOException { + public IrGraphStatistics readStats(GraphId graphId) throws IOException { String statsPath = GraphConfig.GRAPH_STATISTICS.get(configs); return statsPath.isEmpty() ? null : new IrGraphStatistics(new FileInputStream(statsPath)); } diff --git a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java index 2efffb11a9aa..012b373f4c83 100644 --- a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java +++ b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java @@ -18,6 +18,7 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.GraphConfig; +import com.alibaba.graphscope.common.ir.meta.GraphId; import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.reader.IrMetaReader; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; @@ -49,7 +50,7 @@ public IrMeta readMeta() throws IOException { } @Override - public GraphStatistics readStats() throws IOException { + public GraphStatistics readStats(GraphId graphId) throws IOException { // TODO: support statistics, otherwise, the CBO will not work throw new NotImplementedException( "reading graph statistics in vineyard is unimplemented yet"); diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootIrMetaReader.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootIrMetaReader.java index b2ed3d4cac2d..a1a7d5f3ded7 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootIrMetaReader.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootIrMetaReader.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.groot.servers.ir; +import com.alibaba.graphscope.common.ir.meta.GraphId; import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.SnapshotId; import com.alibaba.graphscope.common.ir.meta.reader.IrMetaReader; @@ -48,7 +49,7 @@ public IrMeta readMeta() throws IOException { } @Override - public GraphStatistics readStats() throws IOException { + public GraphStatistics readStats(GraphId graphId) throws IOException { // TODO: add statistics, otherwise, the CBO will not work throw new NotImplementedException("reading graph statistics in groot is unimplemented yet"); } From e9e7b5ae583cfb956b278920000549c24515a672 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Mon, 3 Jun 2024 03:52:22 +0000 Subject: [PATCH 06/25] todo: after merge the get_statistics pr, fix the ci --- .github/workflows/hqps-db-ci.yml | 14 +++ flex/tests/hqps/engine_config_test.yaml | 7 ++ .../tests/hqps/hqps_compiler_get_meta_test.sh | 99 +++++++++++++++++++ flex/tests/hqps/test_count_vertices.py | 38 +++++++ .../ir/meta/fetcher/DynamicIrMetaFetcher.java | 1 + 5 files changed, 159 insertions(+) create mode 100644 flex/tests/hqps/hqps_compiler_get_meta_test.sh create mode 100644 flex/tests/hqps/test_count_vertices.py diff --git a/.github/workflows/hqps-db-ci.yml b/.github/workflows/hqps-db-ci.yml index 76101786e21f..436e1886b945 100644 --- a/.github/workflows/hqps-db-ci.yml +++ b/.github/workflows/hqps-db-ci.yml @@ -180,6 +180,20 @@ jobs: ./tests/hqps/query_test ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \ /tmp/csr-data-dir/ + - name: Test get graph meta from admin service + env: + GS_TEST_DIR: ${{ github.workspace }}/gstest + INTERACTIVE_WORKSPACE: /tmp/interactive_workspace + run: | + cd ${GITHUB_WORKSPACE}/flex/tests/hqps + sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test.yaml + sed -i 's/mode: local/mode: http/g' ./engine_config_test.yaml + pip3 install argparse + pip3 install neo4j >= 4.4.19 + bash hqps_compiler_get_meta_test.sh ${INTERACTIVE_WORKSPACE} ./engine_config_test.yaml + sed -i 's/mode: http/mode: local/g' ./engine_config_test.yaml + sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml + - name: Run codegen test. env: GS_TEST_DIR: ${{ github.workspace }}/gstest diff --git a/flex/tests/hqps/engine_config_test.yaml b/flex/tests/hqps/engine_config_test.yaml index a5d42fca55f2..cd456da50842 100644 --- a/flex/tests/hqps/engine_config_test.yaml +++ b/flex/tests/hqps/engine_config_test.yaml @@ -23,6 +23,13 @@ compiler: - FilterIntoJoinRule - FilterMatchRule - NotMatchToAntiJoinRule + meta: + reader: + mode: local + endpoint: localhost:7777 # only for http mode + interval: + graph_meta: 1000 # ms + statistics: 86400000 # ms endpoint: default_listen_address: localhost bolt_connector: diff --git a/flex/tests/hqps/hqps_compiler_get_meta_test.sh b/flex/tests/hqps/hqps_compiler_get_meta_test.sh new file mode 100644 index 000000000000..48323c81316f --- /dev/null +++ b/flex/tests/hqps/hqps_compiler_get_meta_test.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Copyright 2020 Alibaba Group Holding Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +FLEX_HOME=${SCRIPT_DIR}/../../ +SERVER_BIN=${FLEX_HOME}/build/bin/interactive_server +GIE_HOME=${FLEX_HOME}/../interactive_engine/ +ADMIN_PORT=7777 +QUERY_PORT=10000 + +# +if [ ! $# -eq 2 ]; then + echo "only receives: $# args, need 2" + echo "Usage: $0 " + exit 1 +fi + +INTERACTIVE_WORKSPACE=$1 +ENGINE_CONFIG_PATH=$2 +if [ ! -d ${INTERACTIVE_WORKSPACE} ]; then + echo "INTERACTIVE_WORKSPACE: ${INTERACTIVE_WORKSPACE} not exists" + exit 1 +fi +if [ ! -f ${ENGINE_CONFIG_PATH} ]; then + echo "ENGINE_CONFIG: ${ENGINE_CONFIG_PATH} not exists" + exit 1 +fi + + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color +err() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] -ERROR- $* ${NC}" >&2 +} + +info() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] -INFO- $* ${NC}" +} + + +kill_service(){ + info "Kill Service first" + ps -ef | grep "interactive_server" | awk '{print $2}' | xargs kill -9 || true + ps -ef | grep "compiler" | awk '{print $2}' | xargs kill -9 || true + sleep 3 + # check if service is killed + info "Kill Service success" +} + +# kill service when exit +trap kill_service EXIT + +# start engine service and load ldbc graph +start_engine_service(){ + #check SERVER_BIN exists + if [ ! -f ${SERVER_BIN} ]; then + err "SERVER_BIN not found" + exit 1 + fi + + cmd="${SERVER_BIN} -c ${ENGINE_CONFIG_PATH} --enable-admin-service true " + cmd="${cmd} -w ${INTERACTIVE_WORKSPACE} --start-compiler true" + + echo "Start engine service with command: ${cmd}" + eval ${cmd} + sleep 10 + #check interactive_server is running, if not, exit + ps -ef | grep "interactive_server" | grep -v grep + + info "Start engine service success" +} + +run_cypher_test() { + # run a simple cypher query: MATCH (n) RETURN count(n) + python3 ./test_count_vertices.py --endpoint localhost:7687 +} + +kill_service +start_engine_service +# comiper service will fail to start, if the graph meta can not be retrieved +run_cypher_test +kill_service + + + + diff --git a/flex/tests/hqps/test_count_vertices.py b/flex/tests/hqps/test_count_vertices.py new file mode 100644 index 000000000000..b85761a7cea0 --- /dev/null +++ b/flex/tests/hqps/test_count_vertices.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from neo4j import GraphDatabase +from neo4j import Session as Neo4jSession + +import argparse + +def count_vertices(sess: Neo4jSession): + query = "MATCH (n) RETURN COUNT(n);" + result = sess.run(query) + for record in result: + print(record[0]) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Count the number of vertices in a graph.") + parser.add_argument("--endpoint", type=str, required=True, help="The endpoint to connect.") + args = parser.parse_args() + + driver = GraphDatabase.driver(args.endpoint, auth=None) + with driver.session() as session: + count_vertices(session) + driver.close() \ No newline at end of file diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java index 051f6ed8e17c..b381ef756f78 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java @@ -68,6 +68,7 @@ private synchronized void syncMeta() { IrMeta meta = this.reader.readMeta(); this.currentState = new IrMetaStats( + meta.getGraphId(), meta.getSnapshotId(), meta.getSchema(), meta.getStoredProcedures(), From 83e3a312b57841c031c6b0b55eddc1218d3280d6 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Thu, 6 Jun 2024 20:08:54 +0800 Subject: [PATCH 07/25] add necessary config for irConfigs --- .../alibaba/graphscope/groot/servers/ir/IrServiceProducer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java index f3688962de76..1aea24fcae86 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java @@ -141,6 +141,8 @@ private com.alibaba.graphscope.common.config.Configs getConfigs() { addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_IS_ON.getKey(), configMap); addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_OPT.getKey(), configMap); addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_RULES.getKey(), configMap); + addToConfigMapIfExist(FrontendConfig.PHYSICAL_OPT_CONFIG.getKey(), configMap); + addToConfigMapIfExist(FrontendConfig.PHYSICAL_OPT_CONFIG.getKey(), configMap); return new com.alibaba.graphscope.common.config.Configs(configMap); } From e1292e227c37ecc1791845f0447f6e8b672aa777 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Thu, 6 Jun 2024 20:13:49 +0800 Subject: [PATCH 08/25] add necessary config for irConfigs --- .../alibaba/graphscope/groot/servers/ir/IrServiceProducer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java index 1aea24fcae86..256f737766ce 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java @@ -141,7 +141,7 @@ private com.alibaba.graphscope.common.config.Configs getConfigs() { addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_IS_ON.getKey(), configMap); addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_OPT.getKey(), configMap); addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_RULES.getKey(), configMap); - addToConfigMapIfExist(FrontendConfig.PHYSICAL_OPT_CONFIG.getKey(), configMap); + addToConfigMapIfExist(FrontendConfig.GREMLIN_SCRIPT_LANGUAGE_NAME.getKey(), configMap); addToConfigMapIfExist(FrontendConfig.PHYSICAL_OPT_CONFIG.getKey(), configMap); return new com.alibaba.graphscope.common.config.Configs(configMap); } From dc6ba31d1faad261abfc1eb7e1df9a8ba70bfae4 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 12 Jun 2024 07:37:24 +0000 Subject: [PATCH 09/25] fix ci --- .github/workflows/interactive.yml | 2 +- flex/tests/hqps/hqps_compiler_get_meta_test.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 0225dc7f7122..dc0e4e727f0c 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -189,7 +189,7 @@ jobs: sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test.yaml sed -i 's/mode: local/mode: http/g' ./engine_config_test.yaml pip3 install argparse - pip3 install neo4j >= 4.4.19 + pip3 install neo4j bash hqps_compiler_get_meta_test.sh ${INTERACTIVE_WORKSPACE} ./engine_config_test.yaml sed -i 's/mode: http/mode: local/g' ./engine_config_test.yaml sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml diff --git a/flex/tests/hqps/hqps_compiler_get_meta_test.sh b/flex/tests/hqps/hqps_compiler_get_meta_test.sh index 48323c81316f..4e752ad05c88 100644 --- a/flex/tests/hqps/hqps_compiler_get_meta_test.sh +++ b/flex/tests/hqps/hqps_compiler_get_meta_test.sh @@ -54,7 +54,7 @@ info() { kill_service(){ info "Kill Service first" ps -ef | grep "interactive_server" | awk '{print $2}' | xargs kill -9 || true - ps -ef | grep "compiler" | awk '{print $2}' | xargs kill -9 || true + ps -ef | grep "GraphServer" | awk '{print $2}' | xargs kill -9 || true sleep 3 # check if service is killed info "Kill Service success" @@ -75,7 +75,7 @@ start_engine_service(){ cmd="${cmd} -w ${INTERACTIVE_WORKSPACE} --start-compiler true" echo "Start engine service with command: ${cmd}" - eval ${cmd} + ${cmd} & sleep 10 #check interactive_server is running, if not, exit ps -ef | grep "interactive_server" | grep -v grep @@ -85,7 +85,7 @@ start_engine_service(){ run_cypher_test() { # run a simple cypher query: MATCH (n) RETURN count(n) - python3 ./test_count_vertices.py --endpoint localhost:7687 + python3 ./test_count_vertices.py --endpoint neo4j://localhost:7687 } kill_service From 53eab57e062abe1b7459ae924a03910f1dc3108b Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 13 Jun 2024 13:22:08 +0800 Subject: [PATCH 10/25] [GIE Compiler] refine according to reviews --- .github/workflows/interactive.yml | 2 - flex/tests/hqps/engine_config_test.yaml | 11 +++-- .../compiler/conf/ir.compiler.properties | 25 +++++----- .../com/alibaba/graphscope/GraphServer.java | 20 ++++---- .../channel/MetaServiceChannelFetcher.java | 47 ------------------- .../common/config/FrontendConfig.java | 12 ----- .../graphscope/common/config/GraphConfig.java | 15 ++++-- .../graphscope/common/config/YamlConfigs.java | 29 +++++++----- .../ir/meta/fetcher/DynamicIrMetaFetcher.java | 6 +-- .../ir/meta/reader/HttpIrMetaReader.java | 23 ++++----- .../ir/meta/reader/LocalIrMetaReader.java | 25 +++++++--- .../common/config/YamlConfigTest.java | 2 +- .../alibaba/graphscope/common/ir/Utils.java | 6 +-- .../frontend/VineyardIrMetaReader.java | 3 +- .../groot/servers/ir/GrootMetaFetcher.java | 0 .../groot/servers/ir/IrServiceProducer.java | 14 +++--- 16 files changed, 100 insertions(+), 140 deletions(-) delete mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java delete mode 100644 interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootMetaFetcher.java diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index dc0e4e727f0c..24514a6df0c2 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -187,11 +187,9 @@ jobs: run: | cd ${GITHUB_WORKSPACE}/flex/tests/hqps sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test.yaml - sed -i 's/mode: local/mode: http/g' ./engine_config_test.yaml pip3 install argparse pip3 install neo4j bash hqps_compiler_get_meta_test.sh ${INTERACTIVE_WORKSPACE} ./engine_config_test.yaml - sed -i 's/mode: http/mode: local/g' ./engine_config_test.yaml sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml - name: Run codegen test. diff --git a/flex/tests/hqps/engine_config_test.yaml b/flex/tests/hqps/engine_config_test.yaml index cd456da50842..a11bc928c525 100644 --- a/flex/tests/hqps/engine_config_test.yaml +++ b/flex/tests/hqps/engine_config_test.yaml @@ -25,11 +25,12 @@ compiler: - NotMatchToAntiJoinRule meta: reader: - mode: local - endpoint: localhost:7777 # only for http mode - interval: - graph_meta: 1000 # ms - statistics: 86400000 # ms + schema: + uri: http://localhost:7777/v1/service/status + interval: 1000 # ms + statistics: + uri: http://localhost:7777/v1/graph/%s/statistics + interval: 86400000 # ms endpoint: default_listen_address: localhost bolt_connector: diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties index e2e9cacabcf1..77a5a0ef9176 100644 --- a/interactive_engine/compiler/conf/ir.compiler.properties +++ b/interactive_engine/compiler/conf/ir.compiler.properties @@ -13,16 +13,19 @@ pegasus.hosts: localhost:1234 # set timeout in milliseconds to connect to hiactor service # hiactor.timeout: 6000000 -# graph.schema -graph.schema: ../executor/ir/core/resource/modern_schema.json +# set schema access uri, two formats are supported: +# 1. local file path: file://, i.e. file:///path/to/your/schema.json +# 2. remote http path: http://:/, i.e. http://localhost:8080/schema +# we specifically support reading from a relative path if it is a local file. +graph.meta.schema.uri: ../executor/ir/core/resource/modern_schema.json graph.store: exp graph.planner.is.on: true graph.planner.opt: RBO graph.planner.rules: FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule, ExpandGetVFusionRule -# set file path of glogue input statistics -# graph.statistics: src/test/resources/statistics/modern_statistics.json +# set statistics access uri +graph.meta.statistics.uri: src/test/resources/statistics/modern_statistics.json # set stored procedures directory path # graph.stored.procedures: @@ -62,14 +65,8 @@ calcite.default.charset: UTF-8 # set the max capacity of the result streaming buffer for each query # per.query.stream.buffer.max.capacity: 256 -# set the mode to read ir meta, either from a 'local' file or via remote 'http'. -# ir.meta.reader.mode: local +# set the interval in milliseconds to fetch graph schema +# graph.meta.schema.fetch.interval.ms: 1000 -# if ir.meta.reader.mode is 'http', set the remote ir meta service host -# ir.meta.service.host: localhost:8080 - -# if ir.meta.reader.mode is 'http', set the interval in milliseconds to fetch ir meta -# ir.meta.fetch.interval.ms: 1000 - -# if ir.meta.reader.mode is 'http', set the timeout in milliseconds to fetch ir graph statistics -# ir.statistics.fetch.interval.ms: 86400000l +# set the timeout in milliseconds to fetch graph statistics +# graph.meta.statistics.fetch.interval.ms: 86400000l diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java index 1b8159a3ebf1..946110d660e2 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java @@ -20,7 +20,6 @@ import com.alibaba.graphscope.common.client.channel.ChannelFetcher; import com.alibaba.graphscope.common.client.channel.HostURIChannelFetcher; import com.alibaba.graphscope.common.client.channel.HostsRpcChannelFetcher; -import com.alibaba.graphscope.common.client.channel.MetaServiceChannelFetcher; import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.config.GraphConfig; @@ -51,6 +50,7 @@ import java.io.File; import java.io.IOException; +import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -190,18 +190,14 @@ public static void main(String[] args) throws Exception { private static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker tracker) throws IOException { - String readerMode = FrontendConfig.IR_META_READER_MODE.get(configs); - switch (readerMode) { - case "local": - return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); - case "http": - return new DynamicIrMetaFetcher( - configs, - new HttpIrMetaReader(new MetaServiceChannelFetcher(configs)), - tracker); - default: - throw new IllegalArgumentException("unknown ir meta reader mode: " + readerMode); + URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); + if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { + return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); + } else if (schemaUri.getScheme().equals("http")) { + return new DynamicIrMetaFetcher(configs, new HttpIrMetaReader(configs), tracker); } + throw new IllegalArgumentException( + "unknown graph meta reader mode: " + schemaUri.getScheme()); } private static GraphProperties getTestGraph(Configs configs) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java deleted file mode 100644 index e7bfc5e71637..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/client/channel/MetaServiceChannelFetcher.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * - * * Copyright 2020 Alibaba Group Holding Limited. - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package com.alibaba.graphscope.common.client.channel; - -import com.alibaba.graphscope.common.config.Configs; -import com.alibaba.graphscope.common.config.FrontendConfig; -import com.google.common.collect.ImmutableList; - -import java.net.URI; -import java.util.List; - -// read meta service host from config and return it as URI format -public class MetaServiceChannelFetcher implements ChannelFetcher { - private static final String schema = "http"; - private final Configs graphConfig; - - public MetaServiceChannelFetcher(Configs graphConfig) { - this.graphConfig = graphConfig; - } - - @Override - public List fetch() { - String host = FrontendConfig.IR_META_SERVICE_HOST.get(graphConfig); - return ImmutableList.of(URI.create(schema + "://" + host)); - } - - @Override - public Type getType() { - return Type.HTTP; - } -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java index dd5c17db836c..b4add388a962 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java @@ -60,16 +60,4 @@ public class FrontendConfig { public static final Config PER_QUERY_STREAM_BUFFER_MAX_CAPACITY = Config.intConfig("per.query.stream.buffer.max.capacity", 256); - - public static final Config IR_META_FETCH_INTERVAL_MS = - Config.longConfig("ir.meta.fetch.interval.ms", 1000); - - public static final Config IR_STATISTICS_FETCH_INTERVAL_MS = - Config.longConfig("ir.statistics.fetch.interval.ms", 24 * 3600 * 1000l); - - public static Config IR_META_READER_MODE = - Config.stringConfig("ir.meta.reader.mode", "local"); - - public static Config IR_META_SERVICE_HOST = - Config.stringConfig("ir.meta.service.host", "localhost:8080"); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java index 7ec5e1c48769..aa6aa8b0ac48 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java @@ -17,8 +17,17 @@ package com.alibaba.graphscope.common.config; public class GraphConfig { - public static final Config GRAPH_SCHEMA = Config.stringConfig("graph.schema", "."); - public static final Config GRAPH_STATISTICS = - Config.stringConfig("graph.statistics", ""); + public static final Config GRAPH_META_SCHEMA_URI = + Config.stringConfig("graph.meta.schema.uri", "."); + + public static final Config GRAPH_META_STATISTICS_URI = + Config.stringConfig("graph.meta.statistics.uri", ""); + public static final Config GRAPH_STORE = Config.stringConfig("graph.store", "exp"); + + public static final Config GRAPH_META_SCHEMA_FETCH_INTERVAL_MS = + Config.longConfig("graph.meta.schema.fetch.interval.ms", 1000); + + public static final Config GRAPH_META_STATISTICS_FETCH_INTERVAL_MS = + Config.longConfig("graph.meta.statistics.fetch.interval.ms", 24 * 3600 * 1000l); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index 55f481acbf47..977b78554201 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -25,7 +25,6 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Path; import java.util.Map; public class YamlConfigs extends Configs { @@ -48,23 +47,31 @@ public class YamlConfigs extends Configs { return rules; }) .put( - "graph.schema", + "graph.meta.schema.uri", (Configs configs) -> { // if System.properties contains graph.schema, use it - String schema = System.getProperty("graph.schema"); + String schema = System.getProperty("graph.meta.schema.uri"); if (schema != null) { return schema; } - String workspace = configs.get("directories.workspace"); - String subdir = configs.get("directories.subdirs.data"); - String graphName = configs.get("default_graph"); - if (workspace != null && subdir != null && graphName != null) { - return Path.of(workspace, subdir, graphName, "graph.yaml") - .toString(); - } else { - return null; + return configs.get("compiler.meta.reader.schema.uri"); + }) + .put( + "graph.meta.statistics.uri", + (Configs configs) -> { + String statistics = System.getProperty("graph.meta.statistics.uri"); + if (statistics != null) { + return statistics; } + return configs.get("compiler.meta.reader.statistics.uri"); }) + .put( + "graph.meta.schema.fetch.interval.ms", + (Configs configs) -> configs.get("compiler.meta.reader.schema.interval")) + .put( + "graph.meta.statistics.fetch.interval.ms", + (Configs configs) -> + configs.get("compiler.meta.reader.statistics.interval")) .put( "graph.store", (Configs configs) -> { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java index b381ef756f78..56c744bd9ceb 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java @@ -19,7 +19,7 @@ package com.alibaba.graphscope.common.ir.meta.fetcher; import com.alibaba.graphscope.common.config.Configs; -import com.alibaba.graphscope.common.config.FrontendConfig; +import com.alibaba.graphscope.common.config.GraphConfig; import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.IrMetaStats; import com.alibaba.graphscope.common.ir.meta.IrMetaTracker; @@ -49,12 +49,12 @@ public DynamicIrMetaFetcher(Configs configs, IrMetaReader dataReader, IrMetaTrac this.scheduler.scheduleAtFixedRate( () -> syncMeta(), 0, - FrontendConfig.IR_META_FETCH_INTERVAL_MS.get(configs), + GraphConfig.GRAPH_META_SCHEMA_FETCH_INTERVAL_MS.get(configs), TimeUnit.MILLISECONDS); this.scheduler.scheduleAtFixedRate( () -> syncStats(), 0, - FrontendConfig.IR_STATISTICS_FETCH_INTERVAL_MS.get(configs), + GraphConfig.GRAPH_META_STATISTICS_FETCH_INTERVAL_MS.get(configs), TimeUnit.MILLISECONDS); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java index 0b39cb3b72d2..05ae98478633 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java @@ -18,7 +18,8 @@ package com.alibaba.graphscope.common.ir.meta.reader; -import com.alibaba.graphscope.common.client.channel.ChannelFetcher; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.GraphConfig; import com.alibaba.graphscope.common.ir.meta.GraphId; import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.SnapshotId; @@ -41,25 +42,25 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; -import java.util.List; import java.util.Map; // read ir meta from a remote http service public class HttpIrMetaReader implements IrMetaReader { private static final String CONTENT_TYPE = "Content-Type"; private static final String APPLICATION_JSON = "application/json; utf-8"; - private final ChannelFetcher fetcher; private final HttpClient httpClient; + private final Configs configs; - public HttpIrMetaReader(ChannelFetcher fetcher) { - this.fetcher = fetcher; + public HttpIrMetaReader(Configs configs) { + this.configs = configs; this.httpClient = HttpClient.newBuilder().build(); } @Override public IrMeta readMeta() throws IOException { try { - HttpResponse response = sendRequest("/v1/service/status"); + HttpResponse response = + sendRequest(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); String res = response.body(); Preconditions.checkArgument( response.statusCode() == 200, @@ -90,7 +91,10 @@ public IrGraphStatistics readStats(GraphId graphId) throws IOException { Preconditions.checkArgument( graphId.getId() != null, "graph id should not be null in http meta reader"); HttpResponse response = - sendRequest(String.format("/v1/graph/%s/statistics", graphId.getId())); + sendRequest( + String.format( + GraphConfig.GRAPH_META_STATISTICS_URI.get(configs), + graphId.getId())); String res = response.body(); Preconditions.checkArgument( response.statusCode() == 200, @@ -106,12 +110,9 @@ public IrGraphStatistics readStats(GraphId graphId) throws IOException { private HttpResponse sendRequest(String requestUri) throws IOException, InterruptedException { - List metaServiceHost = fetcher.fetch(); - Preconditions.checkArgument(!metaServiceHost.isEmpty(), "can not get meta service host"); - URI getMetaUri = metaServiceHost.get(0).resolve(requestUri); HttpRequest request = HttpRequest.newBuilder() - .uri(getMetaUri) + .uri(URI.create(requestUri)) .headers(CONTENT_TYPE, APPLICATION_JSON) .GET() .build(); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java index ab667c086bac..7c0db6727816 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java @@ -32,6 +32,8 @@ import java.io.FileInputStream; import java.io.IOException; +import java.net.URI; +import java.nio.file.Path; import java.util.Objects; /** @@ -47,24 +49,33 @@ public LocalIrMetaReader(Configs configs) { @Override public IrMeta readMeta() throws IOException { - String metaPath = + String schemaUri = Objects.requireNonNull( - GraphConfig.GRAPH_SCHEMA.get(configs), "ir meta path not exist"); - FileFormatType formatType = FileUtils.getFormatType(metaPath); + GraphConfig.GRAPH_META_SCHEMA_URI.get(configs), "ir meta path not exist"); + URI schemaURI = URI.create(schemaUri); + Path schemaPath = + (schemaURI.getScheme() == null) ? Path.of(schemaURI.getPath()) : Path.of(schemaURI); + FileFormatType formatType = FileUtils.getFormatType(schemaUri); IrGraphSchema graphSchema = - new IrGraphSchema(new SchemaInputStream(new FileInputStream(metaPath), formatType)); + new IrGraphSchema( + new SchemaInputStream( + new FileInputStream(schemaPath.toFile()), formatType)); IrMeta irMeta = (formatType == FileFormatType.YAML) ? new IrMeta( graphSchema, - new GraphStoredProcedures(new FileInputStream(metaPath), this)) + new GraphStoredProcedures(new FileInputStream(schemaUri), this)) : new IrMeta(graphSchema); return irMeta; } @Override public IrGraphStatistics readStats(GraphId graphId) throws IOException { - String statsPath = GraphConfig.GRAPH_STATISTICS.get(configs); - return statsPath.isEmpty() ? null : new IrGraphStatistics(new FileInputStream(statsPath)); + String statsUri = GraphConfig.GRAPH_META_STATISTICS_URI.get(configs); + if (statsUri.isEmpty()) return null; + URI statsURI = URI.create(statsUri); + Path statsPath = + (statsURI.getScheme() == null) ? Path.of(statsURI.getPath()) : Path.of(statsURI); + return new IrGraphStatistics(new FileInputStream(statsPath.toFile())); } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java index addf56aef907..4914e682ad21 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/config/YamlConfigTest.java @@ -63,7 +63,7 @@ public void pegasus_config_test() throws Exception { Assert.assertEquals(18, (int) PegasusConfig.PEGASUS_OUTPUT_CAPACITY.get(configs)); Assert.assertEquals( "./target/test-classes/config/modern/graph.yaml", - GraphConfig.GRAPH_SCHEMA.get(configs)); + GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); Assert.assertEquals("pegasus", FrontendConfig.ENGINE_TYPE.get(configs)); Assert.assertEquals(false, FrontendConfig.GREMLIN_SERVER_DISABLED.get(configs)); Assert.assertEquals(8003, (int) FrontendConfig.GREMLIN_SERVER_PORT.get(configs)); diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/Utils.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/Utils.java index 0d94f23b6eea..92f3d969715a 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/Utils.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/Utils.java @@ -99,7 +99,7 @@ public static IrMeta mockSchemaMeta(String schemaPath) { Configs configs = new Configs( ImmutableMap.of( - GraphConfig.GRAPH_SCHEMA.getKey(), + GraphConfig.GRAPH_META_SCHEMA_URI.getKey(), schemaResource.toURI().getPath())); return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), null).fetch().get(); } catch (Exception e) { @@ -117,9 +117,9 @@ public static IrMeta mockIrMeta( Configs configs = new Configs( ImmutableMap.of( - GraphConfig.GRAPH_SCHEMA.getKey(), + GraphConfig.GRAPH_META_SCHEMA_URI.getKey(), schemaResource.toURI().getPath(), - GraphConfig.GRAPH_STATISTICS.getKey(), + GraphConfig.GRAPH_META_STATISTICS_URI.getKey(), statisticsResource.toURI().getPath())); return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker).fetch().get(); } catch (Exception e) { diff --git a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java index 012b373f4c83..fa550eec6571 100644 --- a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java +++ b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java @@ -44,7 +44,8 @@ public VineyardIrMetaReader(Configs configs) { public IrMeta readMeta() throws IOException { String schemaString = FileUtils.readFileToString( - new File(GraphConfig.GRAPH_SCHEMA.get(configs)), StandardCharsets.UTF_8); + new File(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)), + StandardCharsets.UTF_8); GraphSchema graphSchema = DefaultGraphSchema.buildSchemaFromJson(schemaString); return new IrMeta(new IrGraphSchema(graphSchema, true)); } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootMetaFetcher.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootMetaFetcher.java deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java index 256f737766ce..c72072367ffd 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java @@ -18,10 +18,7 @@ import com.alibaba.graphscope.GraphServer; import com.alibaba.graphscope.common.client.channel.ChannelFetcher; -import com.alibaba.graphscope.common.config.AuthConfig; -import com.alibaba.graphscope.common.config.FrontendConfig; -import com.alibaba.graphscope.common.config.PegasusConfig; -import com.alibaba.graphscope.common.config.PlannerConfig; +import com.alibaba.graphscope.common.config.*; import com.alibaba.graphscope.common.ir.meta.fetcher.DynamicIrMetaFetcher; import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; @@ -133,10 +130,11 @@ private com.alibaba.graphscope.common.config.Configs getConfigs() { addToConfigMapIfExist(FrontendConfig.FRONTEND_SERVER_NUM.getKey(), configMap); // add frontend qps limit addToConfigMapIfExist(FrontendConfig.QUERY_PER_SECOND_LIMIT.getKey(), configMap); - // add ir meta fetch interval - addToConfigMapIfExist(FrontendConfig.IR_META_FETCH_INTERVAL_MS.getKey(), configMap); - // add ir statistics fetch interval - addToConfigMapIfExist(FrontendConfig.IR_STATISTICS_FETCH_INTERVAL_MS.getKey(), configMap); + // add graph schema fetch interval + addToConfigMapIfExist(GraphConfig.GRAPH_META_SCHEMA_FETCH_INTERVAL_MS.getKey(), configMap); + // add graph statistics fetch interval + addToConfigMapIfExist( + GraphConfig.GRAPH_META_STATISTICS_FETCH_INTERVAL_MS.getKey(), configMap); // add graph planner configs addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_IS_ON.getKey(), configMap); addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_OPT.getKey(), configMap); From 699627420d47dac8ca52277f96f19e5593778ac8 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 13 Jun 2024 16:28:30 +0800 Subject: [PATCH 11/25] fix ci issues --- .../com/alibaba/graphscope/GraphServer.java | 22 ++----------------- .../graphscope/common/ir/meta/GraphId.java | 15 +++++++++++++ .../ir/meta/fetcher/DynamicIrMetaFetcher.java | 8 ++++++- .../common/ir/tools/GraphPlanner.java | 7 ++---- .../graphscope/common/ir/tools/Utils.java | 22 +++++++++++++++++++ .../config/gs_interactive_pegasus.yaml | 4 ++++ 6 files changed, 52 insertions(+), 26 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java index 946110d660e2..b40152a6701d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java @@ -24,12 +24,6 @@ import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.config.GraphConfig; import com.alibaba.graphscope.common.ir.meta.IrMeta; -import com.alibaba.graphscope.common.ir.meta.IrMetaTracker; -import com.alibaba.graphscope.common.ir.meta.fetcher.DynamicIrMetaFetcher; -import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; -import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; -import com.alibaba.graphscope.common.ir.meta.reader.HttpIrMetaReader; -import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader; import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; import com.alibaba.graphscope.common.ir.tools.*; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; @@ -50,7 +44,6 @@ import java.io.File; import java.io.IOException; -import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -177,7 +170,8 @@ public static void main(String[] args) throws Exception { Configs configs = Configs.Factory.create(args[0]); GraphRelOptimizer optimizer = new GraphRelOptimizer(configs); IrMetaQueryCallback queryCallback = - new IrMetaQueryCallback(createIrMetaFetcher(configs, optimizer.getGlogueHolder())); + new IrMetaQueryCallback( + Utils.createIrMetaFetcher(configs, optimizer.getGlogueHolder())); GraphServer server = new GraphServer( configs, @@ -188,18 +182,6 @@ public static void main(String[] args) throws Exception { server.start(); } - private static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker tracker) - throws IOException { - URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); - if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { - return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); - } else if (schemaUri.getScheme().equals("http")) { - return new DynamicIrMetaFetcher(configs, new HttpIrMetaReader(configs), tracker); - } - throw new IllegalArgumentException( - "unknown graph meta reader mode: " + schemaUri.getScheme()); - } - private static GraphProperties getTestGraph(Configs configs) { GraphProperties testGraph; switch (GraphConfig.GRAPH_STORE.get(configs)) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/GraphId.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/GraphId.java index bda93af32ab9..8ddf0d75a511 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/GraphId.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/GraphId.java @@ -18,6 +18,8 @@ package com.alibaba.graphscope.common.ir.meta; +import com.google.common.base.Objects; + import org.checkerframework.checker.nullness.qual.Nullable; /** @@ -36,4 +38,17 @@ public GraphId(T id) { public @Nullable T getId() { return id; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GraphId graphId = (GraphId) o; + return Objects.equal(id, graphId.id); + } + + @Override + public int hashCode() { + return Objects.hashCode(id); + } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java index 56c744bd9ceb..b89e573c65d4 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java @@ -66,13 +66,19 @@ public Optional fetch() { private synchronized void syncMeta() { try { IrMeta meta = this.reader.readMeta(); + // if the graph id is changed, we need to update the statistics + GraphStatistics curStats = + (this.currentState == null + || !this.currentState.getGraphId().equals(meta.getGraphId())) + ? null + : this.currentState.getStatistics(); this.currentState = new IrMetaStats( meta.getGraphId(), meta.getSnapshotId(), meta.getSchema(), meta.getStoredProcedures(), - this.currentState == null ? null : this.currentState.getStatistics()); + curStats); if (this.currentState.getStatistics() == null) { syncStats(); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 4e3f65d1b50a..6f2acd7c8465 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -19,9 +19,8 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.ir.meta.IrMeta; -import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; -import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader; import com.alibaba.graphscope.common.ir.meta.schema.GraphOptSchema; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; import com.alibaba.graphscope.common.ir.planner.GraphIOProcessor; @@ -200,9 +199,7 @@ public static void main(String[] args) throws Exception { } Configs configs = Configs.Factory.create(args[0]); GraphRelOptimizer optimizer = new GraphRelOptimizer(configs); - StaticIrMetaFetcher metaFetcher = - new StaticIrMetaFetcher( - new LocalIrMetaReader(configs), optimizer.getGlogueHolder()); + IrMetaFetcher metaFetcher = Utils.createIrMetaFetcher(configs, optimizer.getGlogueHolder()); String query = FileUtils.readFileToString(new File(args[1]), StandardCharsets.UTF_8); GraphPlanner planner = new GraphPlanner( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java index 3e5c72e610d0..d2b5e793f262 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java @@ -16,6 +16,14 @@ package com.alibaba.graphscope.common.ir.tools; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.GraphConfig; +import com.alibaba.graphscope.common.ir.meta.IrMetaTracker; +import com.alibaba.graphscope.common.ir.meta.fetcher.DynamicIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.reader.HttpIrMetaReader; +import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader; import com.alibaba.graphscope.common.ir.meta.schema.CommonOptTable; import com.alibaba.graphscope.common.ir.rel.CommonTableScan; import com.alibaba.graphscope.common.ir.type.GraphLabelType; @@ -34,6 +42,8 @@ import org.apache.calcite.util.NlsString; import org.apache.calcite.util.Sarg; +import java.io.IOException; +import java.net.URI; import java.util.List; import java.util.Set; @@ -128,4 +138,16 @@ private static String toString(String header, RelNode node, Set dedup) { } return builder.toString(); } + + public static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker tracker) + throws IOException { + URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); + if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { + return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); + } else if (schemaUri.getScheme().equals("http")) { + return new DynamicIrMetaFetcher(configs, new HttpIrMetaReader(configs), tracker); + } + throw new IllegalArgumentException( + "unknown graph meta reader mode: " + schemaUri.getScheme()); + } } diff --git a/interactive_engine/compiler/src/test/resources/config/gs_interactive_pegasus.yaml b/interactive_engine/compiler/src/test/resources/config/gs_interactive_pegasus.yaml index 6a18c3b86e3c..bfb9dc7e8fe7 100644 --- a/interactive_engine/compiler/src/test/resources/config/gs_interactive_pegasus.yaml +++ b/interactive_engine/compiler/src/test/resources/config/gs_interactive_pegasus.yaml @@ -24,6 +24,10 @@ compiler: opt: RBO rules: - FilterMatchRule + meta: + reader: + schema: + uri: ./target/test-classes/config/modern/graph.yaml endpoint: default_listen_address: 0.0.0.0 # default localhost bolt_connector: # for cypher, there may be other connectors, such as bolt_connector, https_connector From 063fa0e185bcf544793c6971cea5bf6c75796d95 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 13 Jun 2024 17:23:07 +0800 Subject: [PATCH 12/25] minor fix --- .../com/alibaba/graphscope/GraphServer.java | 22 +++++++++++++++++-- .../common/ir/tools/GraphPlanner.java | 21 +++++++++++++++++- .../graphscope/common/ir/tools/Utils.java | 22 ------------------- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java index b40152a6701d..946110d660e2 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java @@ -24,6 +24,12 @@ import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.config.GraphConfig; import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.IrMetaTracker; +import com.alibaba.graphscope.common.ir.meta.fetcher.DynamicIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.reader.HttpIrMetaReader; +import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader; import com.alibaba.graphscope.common.ir.planner.GraphRelOptimizer; import com.alibaba.graphscope.common.ir.tools.*; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; @@ -44,6 +50,7 @@ import java.io.File; import java.io.IOException; +import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -170,8 +177,7 @@ public static void main(String[] args) throws Exception { Configs configs = Configs.Factory.create(args[0]); GraphRelOptimizer optimizer = new GraphRelOptimizer(configs); IrMetaQueryCallback queryCallback = - new IrMetaQueryCallback( - Utils.createIrMetaFetcher(configs, optimizer.getGlogueHolder())); + new IrMetaQueryCallback(createIrMetaFetcher(configs, optimizer.getGlogueHolder())); GraphServer server = new GraphServer( configs, @@ -182,6 +188,18 @@ public static void main(String[] args) throws Exception { server.start(); } + private static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker tracker) + throws IOException { + URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); + if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { + return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); + } else if (schemaUri.getScheme().equals("http")) { + return new DynamicIrMetaFetcher(configs, new HttpIrMetaReader(configs), tracker); + } + throw new IllegalArgumentException( + "unknown graph meta reader mode: " + schemaUri.getScheme()); + } + private static GraphProperties getTestGraph(Configs configs) { GraphProperties testGraph; switch (GraphConfig.GRAPH_STORE.get(configs)) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java index 6f2acd7c8465..03eff768d1aa 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -18,9 +18,14 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; +import com.alibaba.graphscope.common.config.GraphConfig; import com.alibaba.graphscope.common.ir.meta.IrMeta; +import com.alibaba.graphscope.common.ir.meta.IrMetaTracker; import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; +import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; import com.alibaba.graphscope.common.ir.meta.procedure.StoredProcedureMeta; +import com.alibaba.graphscope.common.ir.meta.reader.HttpIrMetaReader; +import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader; import com.alibaba.graphscope.common.ir.meta.schema.GraphOptSchema; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; import com.alibaba.graphscope.common.ir.planner.GraphIOProcessor; @@ -48,6 +53,8 @@ import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.Objects; @@ -186,6 +193,18 @@ private static Configs createExtraConfigs(@Nullable String keyValues) { return new Configs(keyValueMap); } + private static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker tracker) + throws IOException { + URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); + if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { + return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); + } else if (schemaUri.getScheme().equals("http")) { + return new StaticIrMetaFetcher(new HttpIrMetaReader(configs), tracker); + } + throw new IllegalArgumentException( + "unknown graph meta reader mode: " + schemaUri.getScheme()); + } + public static void main(String[] args) throws Exception { if (args.length < 4 || args[0].isEmpty() @@ -199,7 +218,7 @@ public static void main(String[] args) throws Exception { } Configs configs = Configs.Factory.create(args[0]); GraphRelOptimizer optimizer = new GraphRelOptimizer(configs); - IrMetaFetcher metaFetcher = Utils.createIrMetaFetcher(configs, optimizer.getGlogueHolder()); + IrMetaFetcher metaFetcher = createIrMetaFetcher(configs, optimizer.getGlogueHolder()); String query = FileUtils.readFileToString(new File(args[1]), StandardCharsets.UTF_8); GraphPlanner planner = new GraphPlanner( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java index d2b5e793f262..3e5c72e610d0 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/Utils.java @@ -16,14 +16,6 @@ package com.alibaba.graphscope.common.ir.tools; -import com.alibaba.graphscope.common.config.Configs; -import com.alibaba.graphscope.common.config.GraphConfig; -import com.alibaba.graphscope.common.ir.meta.IrMetaTracker; -import com.alibaba.graphscope.common.ir.meta.fetcher.DynamicIrMetaFetcher; -import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; -import com.alibaba.graphscope.common.ir.meta.fetcher.StaticIrMetaFetcher; -import com.alibaba.graphscope.common.ir.meta.reader.HttpIrMetaReader; -import com.alibaba.graphscope.common.ir.meta.reader.LocalIrMetaReader; import com.alibaba.graphscope.common.ir.meta.schema.CommonOptTable; import com.alibaba.graphscope.common.ir.rel.CommonTableScan; import com.alibaba.graphscope.common.ir.type.GraphLabelType; @@ -42,8 +34,6 @@ import org.apache.calcite.util.NlsString; import org.apache.calcite.util.Sarg; -import java.io.IOException; -import java.net.URI; import java.util.List; import java.util.Set; @@ -138,16 +128,4 @@ private static String toString(String header, RelNode node, Set dedup) { } return builder.toString(); } - - public static IrMetaFetcher createIrMetaFetcher(Configs configs, IrMetaTracker tracker) - throws IOException { - URI schemaUri = URI.create(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); - if (schemaUri.getScheme() == null || schemaUri.getScheme().equals("file")) { - return new StaticIrMetaFetcher(new LocalIrMetaReader(configs), tracker); - } else if (schemaUri.getScheme().equals("http")) { - return new DynamicIrMetaFetcher(configs, new HttpIrMetaReader(configs), tracker); - } - throw new IllegalArgumentException( - "unknown graph meta reader mode: " + schemaUri.getScheme()); - } } From dcf0f8dc1a951b9f5161099a260cca0ecf27aa65 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 13 Jun 2024 18:19:40 +0800 Subject: [PATCH 13/25] minor fix --- .../src/test/resources/config/gs_interactive_hiactor.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml b/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml index d6c6d0398bff..6176498157e8 100644 --- a/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml +++ b/interactive_engine/compiler/src/test/resources/config/gs_interactive_hiactor.yaml @@ -20,6 +20,10 @@ compiler: opt: RBO rules: - FilterMatchRule + meta: + reader: + schema: + uri: ./target/test-classes/config/modern/graph.yaml endpoint: default_listen_address: 0.0.0.0 # default localhost bolt_connector: # for cypher, there may be other connectors, such as bolt_connector, https_connector From 229d92e1518c2781aa0128fe271ce0707f71131c Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 13 Jun 2024 19:22:29 +0800 Subject: [PATCH 14/25] minor fix --- interactive_engine/compiler/conf/ir.compiler.properties | 2 +- .../com/alibaba/graphscope/common/config/GraphConfig.java | 2 +- .../com/alibaba/graphscope/common/config/YamlConfigs.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties index 77a5a0ef9176..f22b4cdbc607 100644 --- a/interactive_engine/compiler/conf/ir.compiler.properties +++ b/interactive_engine/compiler/conf/ir.compiler.properties @@ -17,7 +17,7 @@ pegasus.hosts: localhost:1234 # 1. local file path: file://, i.e. file:///path/to/your/schema.json # 2. remote http path: http://:/, i.e. http://localhost:8080/schema # we specifically support reading from a relative path if it is a local file. -graph.meta.schema.uri: ../executor/ir/core/resource/modern_schema.json +graph.schema: ../executor/ir/core/resource/modern_schema.json graph.store: exp graph.planner.is.on: true diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java index aa6aa8b0ac48..a09faaa5646e 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java @@ -18,7 +18,7 @@ public class GraphConfig { public static final Config GRAPH_META_SCHEMA_URI = - Config.stringConfig("graph.meta.schema.uri", "."); + Config.stringConfig("graph.schema", "."); public static final Config GRAPH_META_STATISTICS_URI = Config.stringConfig("graph.meta.statistics.uri", ""); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index 977b78554201..a0444019a1be 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -47,10 +47,10 @@ public class YamlConfigs extends Configs { return rules; }) .put( - "graph.meta.schema.uri", + "graph.schema", (Configs configs) -> { // if System.properties contains graph.schema, use it - String schema = System.getProperty("graph.meta.schema.uri"); + String schema = System.getProperty("graph.schema"); if (schema != null) { return schema; } From 77f2bb08a8baeca28a706a17851f9a31383be3bd Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Thu, 13 Jun 2024 13:15:55 +0000 Subject: [PATCH 15/25] fixing ci --- flex/engines/http_server/service/hqps_service.cc | 3 ++- flex/storages/metadata/graph_meta_store.cc | 10 ++++++++++ flex/storages/metadata/graph_meta_store.h | 2 ++ flex/tests/interactive/modern_graph_schema_v0_0.yaml | 9 ++++++--- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/flex/engines/http_server/service/hqps_service.cc b/flex/engines/http_server/service/hqps_service.cc index b7dc7e85d28a..e066d0734170 100644 --- a/flex/engines/http_server/service/hqps_service.cc +++ b/flex/engines/http_server/service/hqps_service.cc @@ -252,7 +252,8 @@ bool HQPSService::start_compiler_subprocess( std::stringstream ss; ss << "java -cp " << interactive_class_path; if (!graph_schema_path.empty()) { - ss << " -Dgraph.schema=" << graph_schema_path; + ss << " -Dgraph.schema=http://localhost:" << service_config_.admin_port + << "/v1/service/status"; } ss << " " << COMPILER_SERVER_CLASS_NAME; ss << " " << service_config_.engine_config_path; diff --git a/flex/storages/metadata/graph_meta_store.cc b/flex/storages/metadata/graph_meta_store.cc index d99d17a9817f..7a202c48bfe2 100644 --- a/flex/storages/metadata/graph_meta_store.cc +++ b/flex/storages/metadata/graph_meta_store.cc @@ -337,6 +337,11 @@ CreateGraphMetaRequest CreateGraphMetaRequest::FromJson( } else { request.creation_time = GetCurrentTimeStamp(); } + if (json.contains("stored_procedures")) { + for (auto& plugin : json["stored_procedures"]) { + request.plugin_metas.push_back(PluginMeta::FromJson(plugin)); + } + } return request; } @@ -351,6 +356,11 @@ std::string CreateGraphMetaRequest::ToString() const { json["data_update_time"] = 0; } json["creation_time"] = creation_time; + json["stored_procedures"] = nlohmann::json::array(); + for (auto& plugin_meta : plugin_metas) { + json["stored_procedures"].push_back( + nlohmann::json::parse(plugin_meta.ToJson())); + } return json.dump(); } diff --git a/flex/storages/metadata/graph_meta_store.h b/flex/storages/metadata/graph_meta_store.h index d42337f13895..f9f710c7e23a 100644 --- a/flex/storages/metadata/graph_meta_store.h +++ b/flex/storages/metadata/graph_meta_store.h @@ -130,6 +130,8 @@ struct CreateGraphMetaRequest { std::optional data_update_time; int64_t creation_time; + std::vector plugin_metas; + static CreateGraphMetaRequest FromJson(const std::string& json_str); std::string ToString() const; diff --git a/flex/tests/interactive/modern_graph_schema_v0_0.yaml b/flex/tests/interactive/modern_graph_schema_v0_0.yaml index 252b29c999b8..0c627425fbb1 100644 --- a/flex/tests/interactive/modern_graph_schema_v0_0.yaml +++ b/flex/tests/interactive/modern_graph_schema_v0_0.yaml @@ -20,7 +20,8 @@ schema: - property_id: 1 property_name: name property_type: - primitive_type: DT_STRING + string: + long_text: - property_id: 2 property_name: age property_type: @@ -39,11 +40,13 @@ schema: - property_id: 1 property_name: name property_type: - primitive_type: DT_STRING + string: + long_text: - property_id: 2 property_name: lang property_type: - primitive_type: DT_STRING + string: + long_text: primary_keys: - id edge_types: From c53a0091aa397369b8352e702e9f938f4b2aa05e Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Fri, 14 Jun 2024 10:13:12 +0800 Subject: [PATCH 16/25] minor refine config --- .../alibaba/graphscope/groot/servers/ir/IrServiceProducer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java index c72072367ffd..61f5835bf556 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java @@ -140,7 +140,7 @@ private com.alibaba.graphscope.common.config.Configs getConfigs() { addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_OPT.getKey(), configMap); addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_RULES.getKey(), configMap); addToConfigMapIfExist(FrontendConfig.GREMLIN_SCRIPT_LANGUAGE_NAME.getKey(), configMap); - addToConfigMapIfExist(FrontendConfig.PHYSICAL_OPT_CONFIG.getKey(), configMap); + addToConfigMapIfExist(FrontendConfig.GRAPH_PHYSICAL_OPT.getKey(), configMap); return new com.alibaba.graphscope.common.config.Configs(configMap); } From 30722391a70b41d130cfed75f773f4020093f5e3 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 14 Jun 2024 10:20:53 +0800 Subject: [PATCH 17/25] fix ci tests --- interactive_engine/compiler/conf/ir.compiler.properties | 2 +- .../com/alibaba/graphscope/common/config/GraphConfig.java | 2 +- .../com/alibaba/graphscope/common/config/YamlConfigs.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties index 60e4b82ba208..6f1d6ae72e2d 100644 --- a/interactive_engine/compiler/conf/ir.compiler.properties +++ b/interactive_engine/compiler/conf/ir.compiler.properties @@ -25,7 +25,7 @@ graph.planner.opt: RBO graph.planner.rules: FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule, ExpandGetVFusionRule # set statistics access uri -graph.meta.statistics.uri: src/test/resources/statistics/modern_statistics.json +graph.statistics: src/test/resources/statistics/modern_statistics.json # set stored procedures directory path # graph.stored.procedures: diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java index a09faaa5646e..deeecba5e5db 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java @@ -21,7 +21,7 @@ public class GraphConfig { Config.stringConfig("graph.schema", "."); public static final Config GRAPH_META_STATISTICS_URI = - Config.stringConfig("graph.meta.statistics.uri", ""); + Config.stringConfig("graph.statistics", ""); public static final Config GRAPH_STORE = Config.stringConfig("graph.store", "exp"); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java index a0444019a1be..4d26b2413529 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java @@ -57,9 +57,9 @@ public class YamlConfigs extends Configs { return configs.get("compiler.meta.reader.schema.uri"); }) .put( - "graph.meta.statistics.uri", + "graph.statistics", (Configs configs) -> { - String statistics = System.getProperty("graph.meta.statistics.uri"); + String statistics = System.getProperty("graph.statistics"); if (statistics != null) { return statistics; } From 8ec04477f1ce0a0cfb128204c0fcb84976b2f074 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Fri, 14 Jun 2024 03:54:42 +0000 Subject: [PATCH 18/25] fixing hqps ci --- .../http_server/actor/admin_actor.act.cc | 59 +++++++++++++++++-- .../python/gs_interactive/client/session.py | 2 +- flex/openapi/openapi_interactive.yaml | 2 + flex/storages/metadata/graph_meta_store.cc | 5 ++ flex/tests/interactive/test_plugin_loading.sh | 18 +++++- 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/flex/engines/http_server/actor/admin_actor.act.cc b/flex/engines/http_server/actor/admin_actor.act.cc index f810e34679a2..682856e793f8 100644 --- a/flex/engines/http_server/actor/admin_actor.act.cc +++ b/flex/engines/http_server/actor/admin_actor.act.cc @@ -485,7 +485,13 @@ seastar::future admin_actor::run_get_graph_meta( add_runnable_info(plugin_meta); } auto& graph_meta = meta_res.value(); - graph_meta.plugin_metas = all_plugin_metas; + // There can also be procedures that builtin in the graph meta. + for (auto& plugin_meta : graph_meta.plugin_metas) { + add_runnable_info(plugin_meta); + } + graph_meta.plugin_metas.insert(graph_meta.plugin_metas.end(), + all_plugin_metas.begin(), + all_plugin_metas.end()); return seastar::make_ready_future( gs::Result(std::move(graph_meta.ToJson()))); } else { @@ -694,6 +700,12 @@ seastar::future admin_actor::get_procedures_by_graph_name( for (auto& plugin_meta : all_plugin_metas) { add_runnable_info(plugin_meta); } + for (auto& plugin_meta : graph_meta_res.value().plugin_metas) { + add_runnable_info(plugin_meta); + } + all_plugin_metas.insert(all_plugin_metas.end(), + graph_meta_res.value().plugin_metas.begin(), + graph_meta_res.value().plugin_metas.end()); return seastar::make_ready_future( gs::Result(to_json_str(all_plugin_metas))); } else { @@ -1123,13 +1135,52 @@ seastar::future admin_actor::service_status( res["bolt_port"] = hqps_service.get_service_config().bolt_port; res["gremlin_port"] = hqps_service.get_service_config().gremlin_port; if (running_graph_res.ok()) { - auto graph_meta = + auto graph_meta_res = metadata_store_->GetGraphMeta(running_graph_res.value()); - if (graph_meta.ok()) { - res["graph"] = nlohmann::json::parse(graph_meta.value().ToJson()); + if (graph_meta_res.ok()) { + auto& graph_meta = graph_meta_res.value(); + // Add the plugin meta. + auto get_all_procedure_res = + metadata_store_->GetAllPluginMeta(running_graph_res.value()); + if (get_all_procedure_res.ok()) { + VLOG(10) << "Successfully get all procedures: " + << get_all_procedure_res.value().size(); + auto& all_plugin_metas = get_all_procedure_res.value(); + VLOG(10) << "original all plugins : " << all_plugin_metas.size(); + for (auto& plugin_meta : all_plugin_metas) { + add_runnable_info(plugin_meta); + } + for (auto& plugin_meta : graph_meta.plugin_metas) { + add_runnable_info(plugin_meta); + } + + VLOG(10) << "original graph meta: " << graph_meta.plugin_metas.size(); + for (auto& plugin_meta : all_plugin_metas) { + if (plugin_meta.runnable) { + graph_meta.plugin_metas.emplace_back(plugin_meta); + } + } + VLOG(10) << "got graph meta: " << graph_meta.ToJson(); + res["graph"] = nlohmann::json::parse(graph_meta.ToJson()); + } else { + LOG(ERROR) << "Fail to get all procedures: " + << get_all_procedure_res.status().error_message(); + return seastar::make_exception_future( + get_all_procedure_res.status()); + } + } else { + LOG(ERROR) << "Fail to get graph meta: " + << graph_meta_res.status().error_message(); + res["graph"] = {}; + return seastar::make_exception_future( + graph_meta_res.status()); } } else { res["graph"] = {}; + LOG(ERROR) << "Fail to get running graph: " + << running_graph_res.status().error_message(); + return seastar::make_exception_future( + running_graph_res.status()); } res["start_time"] = hqps_service.get_start_time(); } else { diff --git a/flex/interactive/sdk/python/gs_interactive/client/session.py b/flex/interactive/sdk/python/gs_interactive/client/session.py index c1d821078bd8..79993c3560b1 100644 --- a/flex/interactive/sdk/python/gs_interactive/client/session.py +++ b/flex/interactive/sdk/python/gs_interactive/client/session.py @@ -325,7 +325,7 @@ def __init__(self, uri: str): # get service port service_status = self.get_service_status() if not service_status.is_ok(): - raise Exception("Failed to get service status") + raise Exception("Failed to get service status: ", service_status.get_status_message()) service_port = service_status.get_value().hqps_port # replace the port in uri uri = uri.split(":") diff --git a/flex/openapi/openapi_interactive.yaml b/flex/openapi/openapi_interactive.yaml index f12f76c8b4d3..42769c014bc6 100644 --- a/flex/openapi/openapi_interactive.yaml +++ b/flex/openapi/openapi_interactive.yaml @@ -1298,6 +1298,8 @@ components: type: boolean creation_time: type: integer + update_time: + type: integer UpdateProcedureRequest: x-body-name: update_procedure_request type: object diff --git a/flex/storages/metadata/graph_meta_store.cc b/flex/storages/metadata/graph_meta_store.cc index 7a202c48bfe2..44801276ab91 100644 --- a/flex/storages/metadata/graph_meta_store.cc +++ b/flex/storages/metadata/graph_meta_store.cc @@ -134,6 +134,9 @@ PluginMeta PluginMeta::FromJson(const nlohmann::json& json) { } if (json.contains("name")) { meta.name = json["name"].get(); + if (meta.id.empty()) { + meta.id = meta.name; + } } if (json.contains("bound_graph")) { meta.bound_graph = json["bound_graph"].get(); @@ -155,6 +158,8 @@ PluginMeta PluginMeta::FromJson(const nlohmann::json& json) { } if (json.contains("type")) { meta.type = json["type"].get(); + } else { + meta.type = "cpp"; // default is cpp } if (json.contains("option")) { meta.setOptionFromJsonString(json["option"].dump()); diff --git a/flex/tests/interactive/test_plugin_loading.sh b/flex/tests/interactive/test_plugin_loading.sh index 3c376ca6379d..d9b75c5fb48a 100644 --- a/flex/tests/interactive/test_plugin_loading.sh +++ b/flex/tests/interactive/test_plugin_loading.sh @@ -30,13 +30,22 @@ kill_service(){ trap kill_service EXIT start_engine_service(){ + # expect one args + if [ $# -lt 1 ]; then + echo "Receives: $# args, need 1 args" + echo "Usage: $0 " + exit 1 + fi + enable_compiler=$1 #check SERVER_BIN exists if [ ! -f ${SERVER_BIN} ]; then err "SERVER_BIN not found" exit 1 fi cmd="${SERVER_BIN} -w ${WORKSPACE} -c ${ENGINE_CONFIG_PATH} --enable-admin-service true" - cmd="${cmd} --start-compiler true" + if [ "${enable_compiler}" == "true" ]; then + cmd="${cmd} --start-compiler true" + fi echo "Start engine service with command: ${cmd}" ${cmd} & @@ -93,7 +102,7 @@ check_procedure_loading_and_calling_via_encoder() { exit 1 fi cp $1 ${WORKSPACE}/data/${GRAPH_NAME}/graph.yaml - start_engine_service + start_engine_service false python3 test_call_proc.py --endpoint http://localhost:7777 --input-format encoder @@ -108,16 +117,19 @@ check_procedure_loading_and_calling_via_cypher_json() { exit 1 fi cp $1 ${WORKSPACE}/data/${GRAPH_NAME}/graph.yaml - start_engine_service + start_engine_service true + sleep 5 python3 test_call_proc.py --endpoint http://localhost:7777 --input-format json kill_service } echo "Testing for schema file: ${SCHEMA_VERSION_00}" +rm -rf ${WORKSPACE}/METADATA/ check_procedure_loading_and_calling_via_encoder ${SCHEMA_VERSION_00} echo "Testing for schema file: ${SCHEMA_VERSION_01}" +rm -rf ${WORKSPACE}/METADATA/ check_procedure_loading_and_calling_via_cypher_json ${SCHEMA_VERSION_01} echo "Test passed for plugin loading and calling" \ No newline at end of file From 076ffec164eff28a8760a33d6720e5ca716d610a Mon Sep 17 00:00:00 2001 From: shirly121 Date: Fri, 14 Jun 2024 11:59:20 +0800 Subject: [PATCH 19/25] minor fix --- interactive_engine/compiler/conf/ir.compiler.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interactive_engine/compiler/conf/ir.compiler.properties b/interactive_engine/compiler/conf/ir.compiler.properties index 6f1d6ae72e2d..992f828a2b2e 100644 --- a/interactive_engine/compiler/conf/ir.compiler.properties +++ b/interactive_engine/compiler/conf/ir.compiler.properties @@ -25,7 +25,7 @@ graph.planner.opt: RBO graph.planner.rules: FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule, ExpandGetVFusionRule # set statistics access uri -graph.statistics: src/test/resources/statistics/modern_statistics.json +# graph.statistics: src/test/resources/statistics/modern_statistics.json # set stored procedures directory path # graph.stored.procedures: From ba4d2d7557b03f645dd0e3d1245b3fc70fd971a5 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Fri, 14 Jun 2024 12:05:41 +0800 Subject: [PATCH 20/25] minor refine --- .../ir/rel/metadata/schema/GlogueSchema.java | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/schema/GlogueSchema.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/schema/GlogueSchema.java index 2d199869c264..9bc470321aba 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/schema/GlogueSchema.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/schema/GlogueSchema.java @@ -58,9 +58,13 @@ public GlogueSchema(GraphSchema graphSchema, GraphStatistics statistics) { edgeTypeCardinality = new HashMap(); for (GraphVertex vertex : graphSchema.getVertexList()) { schemaGraph.addVertex(vertex.getLabelId()); - vertexTypeCardinality.put( - vertex.getLabelId(), - statistics.getVertexTypeCount(vertex.getLabelId()).doubleValue()); + Long vertexTypeCount = statistics.getVertexTypeCount(vertex.getLabelId()); + if (vertexTypeCount == null) { + throw new IllegalArgumentException( + "Vertex type count not found for vertex type: " + vertex.getLabelId()); + } else { + vertexTypeCardinality.put(vertex.getLabelId(), vertexTypeCount.doubleValue()); + } } for (GraphEdge edge : graphSchema.getEdgeList()) { for (EdgeRelation relation : edge.getRelationList()) { @@ -68,14 +72,17 @@ public GlogueSchema(GraphSchema graphSchema, GraphStatistics statistics) { int targetType = relation.getTarget().getLabelId(); EdgeTypeId edgeType = new EdgeTypeId(sourceType, targetType, edge.getLabelId()); schemaGraph.addEdge(sourceType, targetType, edgeType); - edgeTypeCardinality.put( - edgeType, - statistics - .getEdgeTypeCount( - Optional.of(sourceType), - Optional.of(edge.getLabelId()), - Optional.of(targetType)) - .doubleValue()); + Long edgeTypeCount = + statistics.getEdgeTypeCount( + Optional.of(sourceType), + Optional.of(edge.getLabelId()), + Optional.of(targetType)); + if (edgeTypeCount == null) { + throw new IllegalArgumentException( + "Edge type count not found for edge type: " + edge.getLabelId()); + } else { + edgeTypeCardinality.put(edgeType, edgeTypeCount.doubleValue()); + } } } } From aa256f8426874bf2364d990243dbc2f66d6b3a3b Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Fri, 14 Jun 2024 11:06:38 +0000 Subject: [PATCH 21/25] fix packaging --- .gitignore | 1 - flex/interactive/sdk/python/py.typed | 0 2 files changed, 1 deletion(-) create mode 100644 flex/interactive/sdk/python/py.typed diff --git a/.gitignore b/.gitignore index 0a66e390f147..dc6bce24ed8e 100644 --- a/.gitignore +++ b/.gitignore @@ -144,7 +144,6 @@ flex/interactive/sdk/python/gs_interactive/api_client.py flex/interactive/sdk/python/gs_interactive/api_response.py flex/interactive/sdk/python/gs_interactive/configuration.py flex/interactive/sdk/python/gs_interactive/exceptions.py -flex/interactive/sdk/python/gs_interactive/py.typed flex/interactive/sdk/python/gs_interactive/rest.py !flex/interactive/sdk/python/gs_interactive/client/generated/__init__.py diff --git a/flex/interactive/sdk/python/py.typed b/flex/interactive/sdk/python/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 From 4a6c36d487cfea08c0cb7f30595d9e253f28db75 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Mon, 17 Jun 2024 20:41:39 +0800 Subject: [PATCH 22/25] minor update in values.yaml --- charts/graphscope-store/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/graphscope-store/values.yaml b/charts/graphscope-store/values.yaml index 635adc68b9ec..6c35a11f5d01 100644 --- a/charts/graphscope-store/values.yaml +++ b/charts/graphscope-store/values.yaml @@ -489,12 +489,12 @@ zkBasePath: "/graphscope/groot" # gremlinServerPort: 12312 ## GOpt config -## To adopt a CBO planner, set graphPlannerOpt to CBO, set gremlinScriptLanguageName to antlr_gremlin_calcite, and set physicalOptConfig to proto. +## To adopt a CBO planner, set graphPlannerOpt to CBO, set gremlinScriptLanguageName to antlr_gremlin_calcite, and set graphPhysicalOpt to proto. graphPlannerIsOn: true graphPlannerOpt: RBO graphPlannerRules: FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule, ExpandGetVFusionRule gremlinScriptLanguageName: antlr_gremlin_traversal -physicalOptConfig: ffi +graphPhysicalOpt: ffi queryExecutionTimeoutMs: 600000 ## Key-value pair separated by ; From eea8a78db12c32291cc1349008dc61d3caef993c Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Tue, 25 Jun 2024 14:56:39 +0800 Subject: [PATCH 23/25] fix: set snapshot id as the latest in frontend cache, to avoid inconsistency results in query --- .../graphscope/groot/frontend/WrappedSchemaFetcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java index 646fbad49973..54ffe13606d6 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java @@ -43,8 +43,8 @@ public WrappedSchemaFetcher( public Map getSchemaSnapshotPair() { SnapshotWithSchema snapshotSchema = this.snapshotCache.getSnapshotWithSchema(); long MAX_SNAPSHOT_ID = Long.MAX_VALUE - 1; - // Always retrieve the latest result in secondary instance - long snapshotId = isSecondary ? MAX_SNAPSHOT_ID : snapshotSchema.getSnapshotId(); + // Always retrieve the latest snapshot id to avoid inconsistency. + long snapshotId = MAX_SNAPSHOT_ID; GraphSchema schema = snapshotSchema.getGraphDef(); return Map.of(snapshotId, schema); } From bca39bbae4ca47c4ab4f817eb3bd39edbf4ab70c Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 25 Jun 2024 20:41:01 +0800 Subject: [PATCH 24/25] fixing ci Committed-by: xiaolei.zl from Dev container --- .../http_server/actor/admin_actor.act.cc | 5 +- .../sdk/python/test/test_driver.py | 64 ++++++++++++++++++- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/flex/engines/http_server/actor/admin_actor.act.cc b/flex/engines/http_server/actor/admin_actor.act.cc index 682856e793f8..847c6d7b01ca 100644 --- a/flex/engines/http_server/actor/admin_actor.act.cc +++ b/flex/engines/http_server/actor/admin_actor.act.cc @@ -1177,10 +1177,7 @@ seastar::future admin_actor::service_status( } } else { res["graph"] = {}; - LOG(ERROR) << "Fail to get running graph: " - << running_graph_res.status().error_message(); - return seastar::make_exception_future( - running_graph_res.status()); + LOG(INFO) << "No graph is running"; } res["start_time"] = hqps_service.get_start_time(); } else { diff --git a/flex/interactive/sdk/python/test/test_driver.py b/flex/interactive/sdk/python/test/test_driver.py index d0f010d92277..88dc1a782914 100644 --- a/flex/interactive/sdk/python/test/test_driver.py +++ b/flex/interactive/sdk/python/test/test_driver.py @@ -94,7 +94,7 @@ def tearDown(self): print("delete graph: ", rep2) def test_example(self): - self.createGraph() + self._graph_id = self.createGraph() self.bulkLoading() self.waitJobFinish() self.list_graph() @@ -103,6 +103,7 @@ def test_example(self): self.createCypherProcedure() self.createCppProcedure() self.restart() + self.restartOnNewGraph() self.getStatistics() self.callProcedure() self.callProcedureWithHttp() @@ -154,8 +155,7 @@ def createGraph(self): create_graph.var_schema = create_schema resp = self._sess.create_graph(create_graph) assert resp.is_ok() - self._graph_id = resp.get_value().graph_id - print("create graph: ", self._graph_id) + return resp.get_value().graph_id def bulkLoading(self): assert os.environ.get("FLEX_DATA_DIR") is not None @@ -263,6 +263,64 @@ def restart(self): print("restart: ", resp.get_value()) # wait 5 seconds time.sleep(5) + # get service status + resp = self._sess.get_service_status() + assert resp.is_ok() + print("get service status: ", resp.get_value()) + + def restartOnNewGraph(self): + original_graph_id = None + status_res = self._sess.get_service_status() + assert status_res.is_ok() + status = status_res.get_value() + if status.status == "Running": + if status.graph is not None and status.graph.id is not None: + original_graph_id = status.graph.id + else: + raise Exception("service status error, graph id is None") + elif status.status == "Stopped": + pass + else: + raise Exception("service status error " + status) + assert original_graph_id is not None + # create new graph + new_graph_id = self.createGraph() + # start service + print("start service on new graph: ", new_graph_id) + start_service_res = self._sess.start_service( + start_service_request=StartServiceRequest(graph_id=new_graph_id) + ) + assert start_service_res.is_ok() + # restart service + print("restart service on new graph: ", new_graph_id) + restart_res = self._sess.restart_service() + assert restart_res.is_ok() + # get status + print("get service status: ") + status_res = self._sess.get_service_status() + assert status_res.is_ok() + print("get service status: ", status_res.get_value().status) + # stop + print("stop service: ") + stop_res = self._sess.stop_service() + assert stop_res.is_ok() + # get status + print("get service status: ") + status_res = self._sess.get_service_status() + assert status_res.is_ok() + print("get service status: ", status_res.get_value().status) + assert status_res.get_value().status == "Stopped" + # after stop, we should be able to delete the graph + print("delete graph: ", new_graph_id) + delete_res = self._sess.delete_graph(new_graph_id) + assert delete_res.is_ok() + # start on original graph + print("start service on original graph: ", original_graph_id) + start_service_res = self._sess.start_service( + start_service_request=StartServiceRequest(graph_id=original_graph_id) + ) + assert start_service_res.is_ok() + print("finish restartOnNewGraph") def getStatistics(self): resp = self._sess.get_graph_statistics(self._graph_id) From 47360520610d5df40f22aa029781e9443adef9c7 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 26 Jun 2024 12:03:02 +0800 Subject: [PATCH 25/25] fix Committed-by: xiaolei.zl from Dev container --- flex/interactive/sdk/python/test/test_driver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flex/interactive/sdk/python/test/test_driver.py b/flex/interactive/sdk/python/test/test_driver.py index 88dc1a782914..fbadf86f9770 100644 --- a/flex/interactive/sdk/python/test/test_driver.py +++ b/flex/interactive/sdk/python/test/test_driver.py @@ -321,6 +321,7 @@ def restartOnNewGraph(self): ) assert start_service_res.is_ok() print("finish restartOnNewGraph") + time.sleep(5) def getStatistics(self): resp = self._sess.get_graph_statistics(self._graph_id)