diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index ed78524a9f6..f78b7bc91f2 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -33,6 +33,7 @@ env: NPROC: 2 # default Parallel build number for GitHub's Linux runner EXAMPLES_ENABLE: OFF # turn off hybridse's example code HYBRIDSE_TESTING_ENABLE: OFF # turn off hybridse's test code + SPARK_HOME: /tmp/spark/ jobs: java-sdk: @@ -41,9 +42,8 @@ jobs: image: ghcr.io/4paradigm/hybridsql:latest env: SQL_JAVASDK_ENABLE: ON - OPENMLDB_BUILD_TARGET: "cp_native_so openmldb" + OPENMLDB_BUILD_TARGET: "cp_native_so openmldb test_udf" MAVEN_OPTS: -Duser.home=/github/home - SPARK_HOME: /tmp/spark/ steps: - uses: actions/checkout@v2 @@ -59,7 +59,7 @@ jobs: - name: Import GPG key id: import_gpg if: github.event_name == 'push' - uses: crazy-max/ghaction-import-gpg@v4 + uses: crazy-max/ghaction-import-gpg@v5 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} @@ -82,6 +82,11 @@ jobs: run: | make build + - name: build taskmanager + working-directory: java + run: | + ./mvnw package --batch-mode -DskipTests=true -Dscalatest.skip=true -Dwagon.skip=true -Dmaven.test.skip=true + - name: upload linux library if: github.event_name == 'push' uses: actions/upload-artifact@v2 @@ -95,7 +100,9 @@ jobs: run: | sh steps/ut_zookeeper.sh start sh steps/download_openmldb_spark.sh $SPARK_HOME - cd onebox && sh start_onebox.sh && sh start_onebox.sh standalone && cd - || exit + ./onebox/start_onebox.sh + ./onebox/start_onebox.sh standalone + ./onebox/start_onebox.sh taskmanager - name: run java modules smoke test working-directory: java @@ -137,20 +144,18 @@ jobs: - name: stop services run: | - cd onebox && ./stop_all.sh && cd - || exit + ./onebox/stop_all.sh sh steps/ut_zookeeper.sh stop java-sdk-mac: # mac job for java sdk. steps are almost same with job 'java-sdk' # except mvn deploy won't target all modules, just hybridse-native & openmldb-native - # the job only run on tag push or manual workflow dispatch due to no test runs runs-on: macos-latest needs: - java-sdk - if: github.event_name == 'push' env: SQL_JAVASDK_ENABLE: ON - OPENMLDB_BUILD_TARGET: "cp_native_so openmldb" + OPENMLDB_BUILD_TARGET: "cp_native_so openmldb test_udf" NPROC: 3 steps: - uses: actions/checkout@v3 @@ -189,6 +194,7 @@ jobs: - name: Import GPG key id: import_gpg + if: github.event_name == 'push' uses: crazy-max/ghaction-import-gpg@v5 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} @@ -197,6 +203,30 @@ jobs: run: | make build + - name: build taskmanager + working-directory: java + run: | + ./mvnw package --batch-mode -DskipTests=true -Dscalatest.skip=true -Dwagon.skip=true -Dmaven.test.skip=true + + # ======================================== # + # MacOS Tests + # ======================================== # + - name: start services + run: | + sh steps/ut_zookeeper.sh start + sh steps/download_openmldb_spark.sh $SPARK_HOME + ./onebox/start_onebox.sh + ./onebox/start_onebox.sh standalone + ./onebox/start_onebox.sh taskmanager + + - name: run java modules smoke test + working-directory: java + run: | + ./mvnw --batch-mode test + + # ======================================== # + # Deploy java modules + # ======================================== # - name: mvn deploy working-directory: java run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 394ca89e329..e09451e4b6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,5 @@ -cmake_minimum_required (VERSION 3.13) +# RPATH "$ORIGIN" tokens are properly escaped if policy CMP0095 ON, which requires cmake >= 3.16 +cmake_minimum_required (VERSION 3.16) if (POLICY CMP0068) cmake_policy(SET CMP0068 NEW) endif () @@ -97,6 +98,8 @@ option(COVERAGE_NO_DEPS "Coverage without test deps, should ensure test built" O option(SANITIZER_ENABLE "Enable AddressSanitizer in Debug mode" OFF) # add_library can reply on this variable # see https://cmake.org/cmake/help/latest/variable/BUILD_SHARED_LIBS.html#variable:BUILD_SHARED_LIBS +# WARN: installed binaries like openmldb may not runnable since the INSTALL_RPATH is not updated accordingly +# Solution: 1. update INSTALL_RPATH for install target 2. install dependent shared libraries option(BUILD_SHARED_LIBS "Enable build shared Libraries instead static" OFF) option(TESTING_ENABLE_STRIP "Strip test binary that may help reduce storage space" OFF) @@ -140,6 +143,11 @@ include(farmhash) if (CMAKE_SYSTEM_NAME STREQUAL "Linux") set(OS_LIB ${CMAKE_THREAD_LIBS_INIT} rt) set(BRPC_LIBS ${BRPC_LIBRARY} ${Protobuf_LIBRARIES} ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${UNWIND_LIBRARY} ${OPENSSL_LIBRARIES} ${LEVELDB_LIBRARY} ${Z_LIBRARY} ${SNAPPY_LIBRARY} dl pthread ${OS_LIB}) + # OPENMLDB_RPATH + # custom RPATH tag for necessary install binaries (like bin/openmldb) + # relative to binary location so any install location works, binary is + # by convention installed to 'bin/' directory, search to '/../$LIB' and '/../udf' + set(OPENMLDB_RPATH "$ORIGIN/../$LIB" "$ORIGIN/../udf") elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") set(OS_LIB ${CMAKE_THREAD_LIBS_INIT} @@ -153,6 +161,7 @@ elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop") set(BRPC_LIBS ${BRPC_LIBRARY} ${Protobuf_LIBRARIES} ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} ${OPENSSL_LIBRARIES} ${LEVELDB_LIBRARY} ${Z_LIBRARY} ${SNAPPY_LIBRARY} dl pthread ${OS_LIB}) + set(OPENMLDB_RPATH "@loader_path/../lib" "@loader_path/../udf") endif () if (SANITIZER_ENABLE) @@ -309,9 +318,9 @@ install( PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ ) install( - FILES ${OPENMLDB_SBINS} - DESTINATION sbin - PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ + FILES ${OPENMLDB_SBINS} + DESTINATION sbin + PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ ) install(FILES ${OPENMLDB_CONFS} DESTINATION conf) install( diff --git a/hybridse/CMakeLists.txt b/hybridse/CMakeLists.txt index a0fd5478d93..576a0b75bd3 100644 --- a/hybridse/CMakeLists.txt +++ b/hybridse/CMakeLists.txt @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 3.13) +cmake_minimum_required(VERSION 3.16) if (POLICY CMP0068) cmake_policy(SET CMP0068 NEW) diff --git a/hybridse/src/udf/dynamic_lib_manager.cc b/hybridse/src/udf/dynamic_lib_manager.cc index c6a034247cd..d75f213d3b7 100644 --- a/hybridse/src/udf/dynamic_lib_manager.cc +++ b/hybridse/src/udf/dynamic_lib_manager.cc @@ -45,7 +45,7 @@ base::Status DynamicLibManager::ExtractFunction(const std::string& name, bool is } } if (!so_handle) { - void* handle = dlopen(file.c_str(), RTLD_LAZY); + void* handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); if (handle == nullptr) { std::string err_msg; err_msg = "can not open the dynamic library: " + file + ", error: " + dlerror() + ", try to use abs path"; @@ -57,7 +57,7 @@ base::Status DynamicLibManager::ExtractFunction(const std::string& name, bool is return {common::kExternalUDFError, err_msg}; } - handle = dlopen(abs_path_buff, RTLD_LAZY); + handle = dlopen(abs_path_buff, RTLD_LAZY | RTLD_LOCAL); if (handle == nullptr) { err_msg.append("dlopen abs path failed, error: ").append(dlerror()); return {common::kExternalUDFError, err_msg}; diff --git a/java/hybridse-sdk/pom.xml b/java/hybridse-sdk/pom.xml index e9b7a2fc9bd..d9268593816 100644 --- a/java/hybridse-sdk/pom.xml +++ b/java/hybridse-sdk/pom.xml @@ -72,7 +72,7 @@ org.testng testng - 6.14.3 + ${testng.version} test diff --git a/java/openmldb-batch/pom.xml b/java/openmldb-batch/pom.xml index 5fe4dfb31aa..174b84e4c32 100644 --- a/java/openmldb-batch/pom.xml +++ b/java/openmldb-batch/pom.xml @@ -107,7 +107,7 @@ org.testng testng - 6.14.3 + ${testng.version} test diff --git a/java/openmldb-common/pom.xml b/java/openmldb-common/pom.xml index d24d31c44a6..a614b98cb05 100644 --- a/java/openmldb-common/pom.xml +++ b/java/openmldb-common/pom.xml @@ -44,7 +44,7 @@ org.testng testng - 6.14.3 + ${testng.version} test @@ -59,4 +59,4 @@ - \ No newline at end of file + diff --git a/java/openmldb-jdbc/pom.xml b/java/openmldb-jdbc/pom.xml index fe29c9c4905..dd3b576c30f 100644 --- a/java/openmldb-jdbc/pom.xml +++ b/java/openmldb-jdbc/pom.xml @@ -52,14 +52,21 @@ 2.17.2 - org.testng testng - 6.14.3 + ${testng.version} test + + org.apache.commons + commons-lang3 + 3.12.0 + test + + + diff --git a/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/JDBCDriverTest.java b/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/JDBCDriverTest.java index 5c62bca51dc..1ad78233239 100644 --- a/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/JDBCDriverTest.java +++ b/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/JDBCDriverTest.java @@ -60,8 +60,7 @@ public void connection() { connection = DriverManager.getConnection( String.format("jdbc:openmldb:///%s?zk=%s&zkPath=%s&zkLogLevel=0", dbName, zk, zkPath)); } catch (SQLException | ClassNotFoundException e) { - e.printStackTrace(); - Assert.fail("jdbc connection failed"); + Assert.fail("jdbc connection failed", e); } } @@ -162,8 +161,7 @@ public void testForPulsarConnector() throws SQLException { deleteStatement.setInt(1, 1); deleteStatement.execute(); } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); + Assert.fail("catched exception", e); } // useless but won't fail @@ -227,8 +225,7 @@ public void testForKafkaConnector() throws SQLException { .prepareStatement("DELETE FROM " + tableName + " WHERE c1=?"); preparedStatement.setInt(1, 1); } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); + Assert.fail("catched exception", e); } // sink, catalog and schema patterns are always be null diff --git a/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/RequestPreparedStatementTest.java b/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/RequestPreparedStatementTest.java index f761138cb49..428e24de984 100644 --- a/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/RequestPreparedStatementTest.java +++ b/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/RequestPreparedStatementTest.java @@ -113,8 +113,7 @@ public void testRequest() { ok = executor.dropDB(dbname); Assert.assertTrue(ok); } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); + Assert.fail("catched exception", e); } finally { try { if (resultSet != null) { diff --git a/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/SQLRouterSmokeTest.java b/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/SQLRouterSmokeTest.java index 61945b0740d..5eb21f19ea1 100644 --- a/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/SQLRouterSmokeTest.java +++ b/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/SQLRouterSmokeTest.java @@ -42,6 +42,7 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Arrays; +import org.apache.commons.lang3.SystemUtils; public class SQLRouterSmokeTest { public static SqlExecutor clusterExecutor; @@ -65,7 +66,7 @@ public class SQLRouterSmokeTest { standaloneOption.setSessionTimeout(20000); standaloneExecutor = new SqlClusterExecutor(standaloneOption); } catch (Exception e) { - e.printStackTrace(); + Assert.fail("cathed exception", e); } } @@ -261,20 +262,64 @@ public void testSmoke(SqlExecutor router) { ok = router.dropDB(dbname); Assert.assertTrue(ok); } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); + Assert.fail("cathed exception", e); } } @Test(dataProvider = "executor") - public void testParameterizedQueryFail(SqlExecutor router) { + public void testCreateFunction(SqlExecutor router) { + java.sql.Statement statement = router.getStatement(); + + try { + // create function ok + if (SystemUtils.IS_OS_MAC) { + statement.execute("CREATE FUNCTION cut2(x STRING) RETURNS STRING OPTIONS (FILE='libtest_udf.dylib')"); + } else if (SystemUtils.IS_OS_LINUX) { + statement.execute("CREATE FUNCTION cut2(x STRING) RETURNS STRING OPTIONS (FILE='libtest_udf.so')"); + } + Assert.assertTrue(statement.execute("SHOW FUNCTIONS")); + + // queryable + { + // // disabled due to #3405 + // statement.execute("set session execute_mode='offline'"); + // statement.execute("set global sync_job=true"); + // Assert.assertTrue(statement.execute("select cut2('hello')")); + // java.sql.ResultSet resultset = statement.getResultSet(); + // resultset.next(); + // String result = resultset.getString(1); + // Assert.assertEquals(result, "he"); + } + + { + statement.execute("set session execute_mode='online'"); + Assert.assertTrue(statement.execute("select cut2('hello')")); + java.sql.ResultSet resultset = statement.getResultSet(); + resultset.next(); + String result = resultset.getString(1); + Assert.assertEquals(result, "he"); + } + } catch (Exception e) { + Assert.fail("cathed exception", e); + } finally { + // dropable + try { + statement.execute("DROP FUNCTION cut2"); + } catch (Exception e) { + Assert.fail(); + } + } + } + + @Test(dataProvider = "executor", expectedExceptions = java.sql.SQLException.class, expectedExceptionsMessageRegExp = ".*Fail to get parameter type with position 2") + public void testParameterizedQueryFail(SqlExecutor router) throws SQLException{ + String dbname = "SQLRouterSmokeTest" + System.currentTimeMillis(); + String ddl = "create table tsql1010 ( col1 bigint, col2 string, index(key=col2, ts=col1));"; try { - String dbname = "SQLRouterSmokeTest" + System.currentTimeMillis(); // create db router.dropDB(dbname); boolean ok = router.createDB(dbname); Assert.assertTrue(ok); - String ddl = "create table tsql1010 ( col1 bigint, col2 string, index(key=col2, ts=col1));"; // create table ok = router.executeDDL(dbname, ddl); Assert.assertTrue(ok); @@ -289,8 +334,9 @@ public void testParameterizedQueryFail(SqlExecutor router) { Assert.fail("executeQuery is expected to throw exception"); rs4.close(); } - } catch (Exception e) { - e.printStackTrace(); + } finally { + router.executeSQL(dbname, "drop table tsql1010"); + router.dropDB(dbname); } } @@ -467,8 +513,7 @@ public void testInsertPreparedState(SqlExecutor router) { ok = router.dropDB(dbname); Assert.assertTrue(ok); } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); + Assert.fail("cathed exception", e); } } @@ -603,8 +648,7 @@ public void testInsertPreparedStateBatch(SqlExecutor router) { ok = router.dropDB(dbname); Assert.assertTrue(ok); } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); + Assert.fail("cathed exception", e); } } diff --git a/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/StatementTest.java b/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/StatementTest.java index a369fd3fdc8..537f22ca5a9 100644 --- a/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/StatementTest.java +++ b/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/StatementTest.java @@ -121,8 +121,7 @@ public void testExecute() { Assert.assertEquals(result.get(1000L), "hello"); Assert.assertEquals(result.get(1001L), "xxxx"); } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); + Assert.fail("catched exception", e); } finally { try { state.close(); diff --git a/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/TestConfig.java b/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/TestConfig.java index b7018900bbb..7e233e1c8da 100644 --- a/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/TestConfig.java +++ b/java/openmldb-jdbc/src/test/java/com/_4paradigm/openmldb/jdbc/TestConfig.java @@ -17,11 +17,11 @@ package com._4paradigm.openmldb.jdbc; public class TestConfig { -// config for cluster mode - public static String ZK_CLUSTER="127.0.0.1:6181"; - public static String ZK_PATH="/onebox"; + // config for cluster mode + public static String ZK_CLUSTER = "127.0.0.1:6181"; + public static String ZK_PATH = "/onebox"; -// config for standalone mode - public static String HOST="127.0.0.1"; - public static long PORT=6527; + // config for standalone mode + public static String HOST = "127.0.0.1"; + public static long PORT = 6527; } diff --git a/java/openmldb-spark-connector/pom.xml b/java/openmldb-spark-connector/pom.xml index e36ad745b71..6c7c6ffebc4 100644 --- a/java/openmldb-spark-connector/pom.xml +++ b/java/openmldb-spark-connector/pom.xml @@ -41,7 +41,7 @@ org.testng testng - 6.14.3 + ${testng.version} test diff --git a/java/openmldb-taskmanager/pom.xml b/java/openmldb-taskmanager/pom.xml index 108725a9acf..e610b782131 100644 --- a/java/openmldb-taskmanager/pom.xml +++ b/java/openmldb-taskmanager/pom.xml @@ -98,11 +98,10 @@ log4j-to-slf4j 2.13.2 - org.testng testng - 6.14.3 + ${testng.version} test @@ -340,6 +339,7 @@ yyyy.MM.dd HH:mm:ss true true + true diff --git a/java/pom.xml b/java/pom.xml index 1d168284dbc..59f435d668f 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -79,6 +79,8 @@ true + + 7.5.1 @@ -399,6 +401,8 @@ true ${project.build.outputDirectory}/git.properties true + + true diff --git a/onebox/start_onebox.sh b/onebox/start_onebox.sh index 639e409b37c..4f40a98c89b 100755 --- a/onebox/start_onebox.sh +++ b/onebox/start_onebox.sh @@ -15,9 +15,9 @@ # limitations under the License. set -eE -set -x cd "$(dirname "$0")/../" +BASE=$(pwd) # allow producing core files ulimit -c unlimited @@ -28,13 +28,27 @@ if [ ! -r "$OPENMLDB_BIN" ]; then exit 1 fi +# enable dynamic loading searches the 'udf' directory +mkdir -p "$(dirname "$OPENMLDB_BIN")/../udf" + # the subdirectory can be set through the environment variable ONEBOX_WORKDIR WORKSPACE=${ONEBOX_WORKDIR:-onebox/workspace} mkdir -p "$WORKSPACE" +mkdir -p "$WORKSPACE/run/" WORKSPACE=$(cd "$WORKSPACE" && pwd) IP=127.0.0.1 +canonical_file_name() { + local var + var=$1 + while [ $# -gt 1 ]; do + shift + var="$var-${1//:/-}" + done + echo "$var" +} + # Function: cluster_start_component # start a openmldb component for cluster # @param $1 role, (tablet/nameserver) @@ -86,14 +100,21 @@ cluster_start_component() { return 3 fi + local canon_name + canon_name="$(canonical_file_name "$role" "$endpoint")" + # just need extra_opts to split - "$OPENMLDB_BIN" \ + LD_DEBUG=libs "$OPENMLDB_BIN" \ --role="$role" \ --endpoint="$endpoint" \ --openmldb_log_dir="$log_dir" \ --zk_cluster="$zk_end" \ --zk_root_path="$zk_path" \ - "${extra_opts[@]}" + "${extra_opts[@]}" > "$WORKSPACE/$canon_name.log" 2>&1 & + + echo $! > "$WORKSPACE/run/$canon_name.pid" + + echo "started $role at $endpoint" } ZK_CLUSTER=$IP:6181 @@ -109,27 +130,48 @@ TABLET2=$IP:9522 start_cluster() { # first start zookeeper - cluster_start_component tablet "$TABLET0" "$WORKSPACE/logs/tablet0" "$ZK_CLUSTER" "/onebox" "$WORKSPACE/tablet0-binlogs" "$WORKSPACE/recycle_bin0" >"$WORKSPACE/tablet0.log" 2>&1 & + cluster_start_component tablet "$TABLET0" "$WORKSPACE/logs/tablet0" "$ZK_CLUSTER" "/onebox" "$WORKSPACE/tablet0-binlogs" "$WORKSPACE/recycle_bin0" sleep 2 - cluster_start_component tablet "$TABLET1" "$WORKSPACE/logs/tablet1" "$ZK_CLUSTER" "/onebox" "$WORKSPACE/tablet1-binlogs" "$WORKSPACE/recycle_bin1" >"$WORKSPACE/tablet1.log" 2>&1 & + cluster_start_component tablet "$TABLET1" "$WORKSPACE/logs/tablet1" "$ZK_CLUSTER" "/onebox" "$WORKSPACE/tablet1-binlogs" "$WORKSPACE/recycle_bin1" sleep 2 - cluster_start_component tablet "$TABLET2" "$WORKSPACE/logs/tablet2" "$ZK_CLUSTER" "/onebox" "$WORKSPACE/tablet2-binlogs" "$WORKSPACE/recycle_bin2" >"$WORKSPACE/tablet1.log" 2>&1 & + cluster_start_component tablet "$TABLET2" "$WORKSPACE/logs/tablet2" "$ZK_CLUSTER" "/onebox" "$WORKSPACE/tablet2-binlogs" "$WORKSPACE/recycle_bin2" sleep 2 - cluster_start_component nameserver "$NS0" "$WORKSPACE/logs/ns0" "$ZK_CLUSTER" "/onebox" >"$WORKSPACE/ns0.log" 2>&1 & + cluster_start_component nameserver "$NS0" "$WORKSPACE/logs/ns0" "$ZK_CLUSTER" "/onebox" sleep 2 - cluster_start_component nameserver "$NS1" "$WORKSPACE/logs/ns1" "$ZK_CLUSTER" "/onebox" >"$WORKSPACE/ns1.log" 2>&1 & + cluster_start_component nameserver "$NS1" "$WORKSPACE/logs/ns1" "$ZK_CLUSTER" "/onebox" sleep 2 - cluster_start_component nameserver "$NS2" "$WORKSPACE/logs/ns2" "$ZK_CLUSTER" "/onebox" >"$WORKSPACE/ns2.log" 2>&1 & + cluster_start_component nameserver "$NS2" "$WORKSPACE/logs/ns2" "$ZK_CLUSTER" "/onebox" sleep 2 echo "cluster start ok" } +TASKMANGER_DIST=java/openmldb-taskmanager/target/openmldb-taskmanager-binary/ +start_taskmanager() { + if [ ! -d "$TASKMANGER_DIST" ]; then + echo "taskmanager dist directory $TASKMANGER_DIST not built, run './mvnw package --batch-mode -DskipTests=true -Dscalatest.skip=true -Dwagon.skip=true -Dmaven.test.skip=true' in java directory first" + exit 1 + fi + + pushd "$TASKMANGER_DIST" + chmod +x bin/*.sh + # NOTE: taskmanager find shared libraraies in "/udf", where starts taskmanager matters + mkdir -p udf/ + cp -v "$BASE/build/udf/"*.{so,dylib} udf/ || true + cp -v "$BASE/onebox/taskmanager.properties" conf/ + + LD_DEBUG=libs ./bin/taskmanager.sh > "$WORKSPACE/logs/taskmanager.log" 2>&1 & + echo $! > "$WORKSPACE/run/taskmanager.pid" + popd + + echo "started taskmanager" +} + SA_NS=$IP:6527 SA_TABLET=$IP:9921 SA_BINLOG="$WORKSPACE/standalone/binlog" @@ -145,18 +187,20 @@ start_standalone() { mkdir -p "$WORKSPACE/logs/standalone-tb" mkdir -p "$WORKSPACE/logs/standalone-ns" - ./build/bin/openmldb --db_root_path="$SA_BINLOG" \ + LD_DEBUG=libs ./build/bin/openmldb --db_root_path="$SA_BINLOG" \ --recycle_bin_root_path="$SA_RECYCLE" \ --openmldb_log_dir="$WORKSPACE/logs/standalone-tb" \ --endpoint="$SA_TABLET" --role=tablet \ --binlog_notify_on_put=true >"$WORKSPACE/sa-tablet.log" 2>&1 & + echo $! > "$WORKSPACE/run/$(canonical_file_name sa-tablet).pid" sleep 2 # start ns - ./build/bin/openmldb --endpoint="$SA_NS" --role=nameserver \ + LD_DEBUG=libs ./build/bin/openmldb --endpoint="$SA_NS" --role=nameserver \ --tablet="$SA_TABLET" \ --openmldb_log_dir="$WORKSPACE/logs/standalone-ns" \ --tablet_offline_check_interval=1 --tablet_heartbeat_timeout=1 >"$WORKSPACE/sa-ns.log" 2>&1 & + echo $! > "$WORKSPACE/run/$(canonical_file_name sa-ns).pid" sleep 2 echo "standalone start ok" } @@ -185,6 +229,10 @@ cluster) standalone) start_standalone ;; +taskmanager) + start_taskmanager + ;; + -h | help) help ;; diff --git a/onebox/stop_all.sh b/onebox/stop_all.sh index 747adcdf929..c9fc245ec22 100755 --- a/onebox/stop_all.sh +++ b/onebox/stop_all.sh @@ -14,11 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x -e +set -e +shopt -s nullglob -if [[ "$OSTYPE" = "darwin"* ]]; then - pkill -9 -x -l openmldb -else - pgrep -a -f "openmldb.*onebox.*" | awk '{print $1}' | xargs -I {} kill -9 {} -fi +cd "$(dirname "$0")" + +for file in workspace/run/*.pid ; do + if [[ "$OSTYPE" = "darwin"* ]]; then + pkill -9 -l -F "$file" || echo "pidfile $file not valid, ignoreing" + else + pkill -9 -e -F "$file" || echo "pidfile $file not valid, ignoreing" + fi + rm -f "$file" +done diff --git a/onebox/taskmanager.properties b/onebox/taskmanager.properties new file mode 100644 index 00000000000..135b944a9c4 --- /dev/null +++ b/onebox/taskmanager.properties @@ -0,0 +1,34 @@ +# Server Config +server.host=127.0.0.1 +server.port=9902 +server.worker_threads=4 +server.io_threads=4 +server.channel_keep_alive_time=1800 +prefetch.jobid.num=1 +job.log.path=./logs/ +external.function.dir=./udf/ +track.unfinished.jobs=true +job.tracker.interval=30 + +# OpenMLDB Config +zookeeper.cluster=127.0.0.1:6181 +zookeeper.root_path=/onebox +zookeeper.session_timeout=5000 +zookeeper.connection_timeout=5000 +zookeeper.max_retries=10 +zookeeper.base_sleep_time=1000 +zookeeper.max_connect_waitTime=30000 + +# Spark Config +spark.home= +spark.master=local[*] +spark.yarn.jars= +spark.default.conf= +spark.eventLog.dir= +spark.yarn.maxAppAttempts=1 + +# relative path, based on which directory taskmanager starts +batchjob.jar.path=./lib/openmldb-batchjob-0.7.0-SNAPSHOT.jar +namenode.uri= +offline.data.prefix=file:///tmp/openmldb_offline_storage/ +hadoop.conf.dir= diff --git a/release/bin/start.sh b/release/bin/start.sh index 4f1c1260b29..acc314a804d 100755 --- a/release/bin/start.sh +++ b/release/bin/start.sh @@ -30,9 +30,6 @@ fi cd "$(dirname "$0")"/../ || exit 1 ROOTDIR=$(pwd) BINDIR=$(pwd)/bin # the script dir -# udf is /udf -LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$(pwd)/udf" -export LD_LIBRARY_PATH RED='\E[1;31m' RES='\E[0m' diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b2edc2d417f..c51a7f6371d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -151,9 +151,14 @@ if(TESTING_ENABLE) compile_test(apiserver) # abs path compile_test_with_extra(datacollector ${CMAKE_CURRENT_SOURCE_DIR}/datacollector/data_collector.cc) - add_library(test_udf SHARED examples/test_udf.cc) endif() +add_library(test_udf SHARED examples/test_udf.cc) +set_target_properties(test_udf + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/udf +) + add_executable(parse_log tools/parse_log.cc $) add_executable(data_exporter tools/data_exporter.cc tools/log_exporter.cc tools/tablemeta_reader.cc $) @@ -172,6 +177,11 @@ target_link_libraries(data_exporter ${EXPORTER_LIBS}) add_executable(openmldb cmd/openmldb.cc base/linenoise.cc) # tablet needs std::filesystem(stdc++fs) target_link_libraries(openmldb ${EXPORTER_LIBS}) +set_target_properties( + openmldb + PROPERTIES + BUILD_RPATH "${OPENMLDB_RPATH}" + INSTALL_RPATH "${OPENMLDB_RPATH}") add_executable(data_collector datacollector/data_collector_server.cc datacollector/data_collector.cc) target_link_libraries(data_collector ${EXPORTER_LIBS}) diff --git a/src/cmd/single_tablet_test.cc b/src/cmd/single_tablet_test.cc index be2335c8cc5..b4928371f0b 100644 --- a/src/cmd/single_tablet_test.cc +++ b/src/cmd/single_tablet_test.cc @@ -75,7 +75,7 @@ TEST_P(DBSDKTest, CreateFunction) { ProcessSQLs(sr_2, {"set @@execute_mode = 'online'"}); } hybridse::sdk::Status status; - std::string so_path = openmldb::test::GetParentDir(openmldb::test::GetExeDir()) + "/libtest_udf.so"; + std::string so_path = openmldb::test::GetParentDir(openmldb::test::GetExeDir()) + "/../udf/libtest_udf.so"; std::string cut2_sql = absl::StrCat("CREATE FUNCTION cut2(x STRING) RETURNS STRING " "OPTIONS (FILE='", so_path, "');"); std::string strlength_sql = absl::StrCat("CREATE FUNCTION strlength(x STRING) RETURNS INT " @@ -150,7 +150,7 @@ TEST_P(DBSDKTest, CreateUdafFunction) { ProcessSQLs(sr_2.get(), {"set @@execute_mode = 'online'"}); } hybridse::sdk::Status status; - std::string so_path = openmldb::test::GetParentDir(openmldb::test::GetExeDir()) + "/libtest_udf.so"; + std::string so_path = openmldb::test::GetParentDir(openmldb::test::GetExeDir()) + "/../udf/libtest_udf.so"; std::string agg_fun_str = absl::StrCat("CREATE AGGREGATE FUNCTION special_sum(x BIGINT) RETURNS BIGINT " "OPTIONS (FILE='", so_path, "');"); std::string agg_fun_str1 = absl::StrCat("CREATE AGGREGATE FUNCTION count_null(x STRING) RETURNS BIGINT " diff --git a/src/sdk/CMakeLists.txt b/src/sdk/CMakeLists.txt index cc959f6a23b..a404783c605 100644 --- a/src/sdk/CMakeLists.txt +++ b/src/sdk/CMakeLists.txt @@ -161,6 +161,9 @@ if(SQL_JAVASDK_ENABLE) SOURCES sql_router_sdk.i) target_include_directories(sql_jsdk PRIVATE ${JNI_INCLUDE_DIRS}) target_compile_options(sql_jsdk PRIVATE -w) + set_target_properties(sql_jsdk + PROPERTIES + BUILD_RPATH "${OPENMLDB_RPATH}") add_dependencies(sql_jsdk hybridse_jsdk_core_static) if(APPLE) diff --git a/steps/download_openmldb_spark.sh b/steps/download_openmldb_spark.sh index 3c48264dd2d..1797a22a83d 100755 --- a/steps/download_openmldb_spark.sh +++ b/steps/download_openmldb_spark.sh @@ -6,7 +6,7 @@ SPARK_HOME_PATH=$1 mkdir -p "$SPARK_HOME_PATH" # Download OpenMLDB Spark distribution -curl -L -o spark-3.0.0-bin-openmldbspark.tgz https://github.com/4paradigm/spark/releases/download/v3.0.0-openmldb0.4.0/spark-3.0.0-bin-openmldbspark.tgz +curl -L -o spark-3.0.0-bin-openmldbspark.tgz https://github.com/4paradigm/spark/releases/download/v3.2.1-openmldb0.8.2/spark-3.2.1-bin-openmldbspark.tgz tar xzf spark-3.0.0-bin-openmldbspark.tgz -C "$SPARK_HOME_PATH" --strip-components=1