feat(interactive): Implement a c++ wrapper for compiler's GraphPlanner

.github/workflows/interactive.yml

@@ -317,6 +317,25 @@ jobs:
         cd ${GITHUB_WORKSPACE}/interactive_engine/
         mvn clean install -Pexperimental -DskipTests -q
 
+    - name: Test physical plan generation
+      run: |
+        cd ${GITHUB_WORKSPACE}/interactive_engine/compiler
+        cat > /tmp/physical_plan_gen_config.yaml <<EOF
+        compiler:
+          planner:
+            is_on: true
+            opt: CBO
+            rules:
+              - FilterIntoJoinRule
+              - FilterMatchRule
+              - NotMatchToAntiJoinRule
+          physical.opt.config: proto
+        EOF
+        echo "  meta.reader.schema.uri: ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml" >> /tmp/physical_plan_gen_config.yaml
+        echo "  meta.reader.statistics.uri: ${GITHUB_WORKSPACE}/interactive_engine/compiler/src/test/resources/statistics/modern_statistics.json" >> /tmp/physical_plan_gen_config.yaml
+        mvn clean install -DskipTests -Pgraph-planner-jni
+        ./target/native/test_graph_planner
+
     - name: Run End-to-End cypher adhoc ldbc query test
       env:
         GS_TEST_DIR: ${{ github.workspace }}/gstest

flex/tests/CMakeLists.txt

@@ -1,2 +1,2 @@
 add_subdirectory(hqps)
-add_subdirectory(rt_mutable_graph)
+add_subdirectory(rt_mutable_graph)

interactive_engine/assembly/graph-planner-jni.xml

@@ -0,0 +1,72 @@
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
+    <id>graph-planner-jni</id>
+    <formats>
+        <format>tar.gz</format>
+    </formats>
+
+    <fileSets>
+        <fileSet>
+            <directory>${project.parent.basedir}/executor/ir/target/release</directory>
+            <includes>
+                <include>libir_core.*</include>
+            </includes>
+            <outputDirectory>native</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/target/native/</directory>
+            <includes>
+                <include>libgraph_planner.*</include>
+            </includes>
+            <outputDirectory>native</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/target/native/</directory>
+            <includes>
+                <include>test_graph_planner</include>
+            </includes>
+            <outputDirectory>bin</outputDirectory>
+            <fileMode>0755</fileMode>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/compiler/target/libs/</directory>
+            <outputDirectory>libs</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/compiler/target/</directory>
+            <includes>
+                <include>compiler-0.0.1-SNAPSHOT.jar</include>
+            </includes>
+            <outputDirectory>libs</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/compiler/conf</directory>
+            <outputDirectory>conf</outputDirectory>
+            <includes>
+                <include>*</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/../flex/interactive/examples/modern_graph/</directory>
+            <outputDirectory>conf</outputDirectory>
+            <includes>
+                <include>graph.yaml</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/compiler/src/test/resources/statistics/</directory>
+            <outputDirectory>conf</outputDirectory>
+            <includes>
+                <include>modern_statistics.json</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/compiler/src/test/resources/config/</directory>
+            <outputDirectory>conf</outputDirectory>
+            <includes>
+                <include>interactive_config_test.yaml</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+</assembly>

interactive_engine/assembly/pom.xml

@@ -71,5 +71,22 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+      <id>graph-planner-jni</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-assembly-plugin</artifactId>
+            <configuration>
+              <finalName>graph-planner-jni</finalName>
+              <descriptors>
+                <descriptor>graph-planner-jni.xml</descriptor>
+              </descriptors>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 </project>

interactive_engine/compiler/build_ir_core.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
 
-set -e
-set -x
-
-if $(! command -v cargo &> /dev/null)
-then
-    echo "cargo not exit, skip compile"
-else
-    cd ../executor/ir/core
-    cargo build --release
-fi
+#set -e
+#set -x
+#
+#if $(! command -v cargo &> /dev/null)
+#then
+#    echo "cargo not exit, skip compile"
+#else
+#    cd ../executor/ir/core
+#    cargo build --release
+#fi

interactive_engine/compiler/build_native.xml

@@ -0,0 +1,51 @@
+<!--
+  ~ Copyright 2021 Alibaba Group Holding Limited.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~  	http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project name="COMPILER-JNI">
+  <target name="graph_planner">
+    <condition property="platform" value="linux64">
+      <os arch="amd64" family="unix"/>
+    </condition>
+
+    <condition property="platform" value="mac">
+      <!--Both m1 and x86 are ok, we will diff them in cmakelist-->
+      <os family="mac"/>
+    </condition>
+
+    <condition property="platform" value="linux64">
+      <os arch="aarch64" family="unix"/>
+    </condition>
+
+    <fail unless="platform" message="Not a supported platform."/>
+
+    <echo level="info" message="Make compiler graph planner jni code"/>
+
+    <echo level="info" message="Native Library Name: graph_planner"/>
+
+    <mkdir dir="${project.build.directory}/native"/>
+
+    <exec dir="${project.build.directory}/native" executable="cmake" failonerror="true">
+        <arg line="${basedir}/compiler/src/main/native"/>
+    </exec>
+
+    <exec dir="${project.build.directory}/native" executable="make" failonerror="true">
+      <arg line="VERBOSE=1 -j4"/>
+    </exec>
+
+    <exec dir="${project.build.directory}/native" executable="strip" failonerror="false">
+      <arg line="${project.build.directory}/native/libgraph_planner.so"/>
+    </exec>
+  </target>
+</project>

interactive_engine/compiler/conf/application.yaml

@@ -0,0 +1,22 @@
+#
+# /*
+#  * Copyright 2020 Alibaba Group Holding Limited.
+#  *
+#  * Licensed under the Apache License, Version 2.0 (the "License");
+#  * you may not use this file except in compliance with the License.
+#  * You may obtain a copy of the License at
+#  *
+#  * http://www.apache.org/licenses/LICENSE-2.0
+#  *
+#  * Unless required by applicable law or agreed to in writing, software
+#  * distributed under the License is distributed on an "AS IS" BASIS,
+#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  * See the License for the specific language governing permissions and
+#  * limitations under the License.
+#  */
+#
+
+server:
+  port: 8080
+app:
+  name: GraphPlannerService

interactive_engine/compiler/conf/ir.compiler.properties

@@ -21,11 +21,11 @@ graph.schema: ../executor/ir/core/resource/modern_schema.json
 graph.store: exp
 
 graph.planner.is.on: true
-graph.planner.opt: RBO
+graph.planner.opt: CBO
 graph.planner.rules: FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule, ExpandGetVFusionRule
 
 # set statistics access uri
-# graph.statistics: src/test/resources/statistics/modern_statistics.json
+graph.statistics: src/test/resources/statistics/modern_statistics.json
 
 # set stored procedures directory path
 # graph.stored.procedures: <your stored procedures directory path>
@@ -60,7 +60,7 @@ calcite.default.charset: UTF-8
 # gremlin.script.language.name: antlr_gremlin_traversal
 
 # the output plan format, can be ffi(default) or proto
-# graph.physical.opt: ffi
+graph.physical.opt: proto
 
 # set the max capacity of the result streaming buffer for each query
 # per.query.stream.buffer.max.capacity: 256

interactive_engine/compiler/pom.xml

@@ -1,3 +1,3 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
@@ -171,6 +171,10 @@
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
     </dependencies>
 
     <profiles>
@@ -294,6 +298,7 @@
                 <artifactId>build-helper-maven-plugin</artifactId>
                 <executions>
                     <execution>
+                        <id>generate-sources</id>
                         <phase>generate-sources</phase>
                         <goals>
                             <goal>add-source</goal>

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java

@@ -210,8 +210,8 @@ private static GraphProperties getTestGraph(Configs configs) {
             case "rust-mcsr":
                 testGraph = TestGraphFactory.RUST_MCSR;
                 break;
-            case "cpp-mcsr":
-                logger.info("using cpp-mcsr as test graph");
+            case "native-mcsr":
+                logger.info("using native-mcsr as test graph");
                 testGraph = TestGraphFactory.CPP_MCSR;
                 break;
             default:

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/YamlConfigs.java

@@ -37,6 +37,9 @@ public class YamlConfigs extends Configs {
                         "graph.planner.is.on",
                         (Configs configs) -> configs.get("compiler.planner.is_on"))
                 .put("graph.planner.opt", (Configs configs) -> configs.get("compiler.planner.opt"))
+                .put(
+                        "graph.planner.cbo.glogue.size",
+                        (Configs configs) -> configs.get("compiler.planner.cbo.glogue.size"))
                 .put(
                         "graph.planner.rules",
                         (Configs configs) -> {
@@ -78,7 +81,7 @@ public class YamlConfigs extends Configs {
                             if (configs.get("compute_engine.store.type") != null) {
                                 return configs.get("compute_engine.store.type");
                             } else {
-                                return "cpp-mcsr";
+                                return "native-mcsr";
                             }
                         })
                 .put(

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/StaticIrMetaFetcher.java

@@ -51,7 +51,7 @@ public StaticIrMetaFetcher(IrMetaReader dataReader, IrMetaTracker tracker) throw
         try {
             return this.reader.readStats(meta.getGraphId());
         } catch (Exception e) {
-            logger.warn("failed to read graph statistics, error is {}", e);
+            logger.warn("failed to read graph statistics, error is: " + e);
             return null;
         }
     }

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/procedure/StoredProcedureMeta.java

@@ -163,16 +163,19 @@ public int hashCode() {
     }
 
     public static class Serializer {
-        public static void perform(StoredProcedureMeta meta, OutputStream outputStream)
+        public static void perform(
+                StoredProcedureMeta meta, OutputStream outputStream, boolean throwsOnFail)
                 throws IOException {
             Yaml yaml = new Yaml();
-            String mapStr = yaml.dump(createProduceMetaMap(meta));
+            String mapStr = yaml.dump(createProduceMetaMap(meta, throwsOnFail));
             outputStream.write(mapStr.getBytes(StandardCharsets.UTF_8));
         }
 
-        private static Map<String, Object> createProduceMetaMap(StoredProcedureMeta meta) {
+        private static Map<String, Object> createProduceMetaMap(
+                StoredProcedureMeta meta, boolean throwsOnFail) {
             GSDataTypeConvertor<RelDataType> typeConvertor =
-                    GSDataTypeConvertor.Factory.create(RelDataType.class, typeFactory);
+                    GSDataTypeConvertor.Factory.create(
+                            RelDataType.class, typeFactory, throwsOnFail);
             return ImmutableMap.of(
                     Config.NAME.getKey(),
                     meta.name,
@@ -217,7 +220,7 @@ private static Map<String, Object> createProduceMetaMap(StoredProcedureMeta meta
     public static class Deserializer {
         public static StoredProcedureMeta perform(InputStream inputStream) throws IOException {
             GSDataTypeConvertor<RelDataType> typeConvertor =
-                    GSDataTypeConvertor.Factory.create(RelDataType.class, typeFactory);
+                    GSDataTypeConvertor.Factory.create(RelDataType.class, typeFactory, true);
             Yaml yaml = new Yaml();
             Map<String, Object> config = yaml.load(inputStream);
             return new StoredProcedureMeta(
@@ -346,7 +349,7 @@ public static class Config {
         // option configurations.
         public static final com.alibaba.graphscope.common.config.Config<String> TYPE =
                 com.alibaba.graphscope.common.config.Config.stringConfig(
-                        "type", "UNKNOWN"); // cypher or cpp
+                        "type", "UNKNOWN"); // cypher or native
         public static final com.alibaba.graphscope.common.config.Config<String> QUERY =
                 com.alibaba.graphscope.common.config.Config.stringConfig("query", "UNKNOWN");
     }