Merge branch 'main' into kanzhen/rw-oauth

.github/workflows/cherry-pick-to-release-branch.yml

@@ -6,6 +6,22 @@ on:
     types: ["closed", "labeled"]
 
 jobs:
+  release_pull_request_1_7_standalone:
+    if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.7-standalone')  &&  github.event.pull_request.merged == true"
+    runs-on: ubuntu-latest
+    name: release_pull_request
+    steps:
+      - name: checkout
+        uses: actions/checkout@v1
+      - name: Create PR to branch
+        uses: risingwavelabs/github-action-cherry-pick@master
+        with:
+          pr_branch: 'release-1.7.0-standalone'
+          pr_labels: 'cherry-pick'
+          pr_body: ${{ format('Cherry picking \#{0} onto branch release-1.7.0-standalone', github.event.number) }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   release_pull_request_1_7:
     if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.7')  &&  github.event.pull_request.merged == true"
     runs-on: ubuntu-latest

Makefile.toml

@@ -29,7 +29,6 @@ env_scripts = [
 set_env ENABLE_TELEMETRY "false"
 
 is_sanitizer_enabled = get_env ENABLE_SANITIZER
-is_all_in_one_enabled = get_env ENABLE_ALL_IN_ONE
 is_hdfs_backend = get_env ENABLE_HDFS
 is_release = get_env ENABLE_RELEASE_PROFILE
 is_not_release = not ${is_release}
@@ -46,13 +45,8 @@ else
   set_env RISEDEV_BUILD_TARGET_DIR ""
 end
 
-if ${is_all_in_one_enabled}
-  set_env RISEDEV_CARGO_BUILD_CRATE "risingwave_cmd_all"
-  set_env RISEDEV_CTL_RUN_CMD "-- risectl"
-else
-  set_env RISEDEV_CARGO_BUILD_CRATE "risingwave_cmd"
-  set_env RISEDEV_CTL_RUN_CMD "--bin risectl --"
-end
+set_env RISEDEV_CARGO_BUILD_CRATE "risingwave_cmd_all"
+set_env RISEDEV_CTL_RUN_CMD "-- risectl"
 
 if ${is_hdfs_backend}
   set_env BUILD_HDFS_BACKEND_CMD "-p risingwave_object_store --features hdfs-backend"
@@ -268,31 +262,10 @@ set -e
 python -mwebbrowser file://$(pwd)/target/doc/index.html
 '''
 
-[tasks.link-standalone-binaries]
-private = true
-category = "RiseDev - Build"
-description = "Link standalone cmds to RiseDev bin"
-condition = { env_not_set = ["ENABLE_ALL_IN_ONE"] }
-script = '''
-#!/usr/bin/env bash
-set -e
-rm -f "${PREFIX_BIN}/compute-node"
-rm -f "${PREFIX_BIN}/meta-node"
-rm -f "${PREFIX_BIN}/frontend"
-rm -f "${PREFIX_BIN}/compactor"
-rm -f "${PREFIX_BIN}/risectl"
-ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/compute-node" "${PREFIX_BIN}/compute-node"
-ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/meta-node" "${PREFIX_BIN}/meta-node"
-ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/frontend" "${PREFIX_BIN}/frontend"
-ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/compactor" "${PREFIX_BIN}/compactor"
-ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risectl" "${PREFIX_BIN}/risectl"
-'''
-
 [tasks.link-all-in-one-binaries]
 private = true
 category = "RiseDev - Build"
 description = "Link all-in-one cmds to RiseDev bin"
-condition = { env_set = ["ENABLE_ALL_IN_ONE"] }
 script = '''
 #!/usr/bin/env bash
 set -e
@@ -339,13 +312,7 @@ script = '''
 #!/usr/bin/env bash
 set -e
 
-binaries=()
-
-if [[ "$ENABLE_ALL_IN_ONE" == "true" ]]; then
-  binaries=("risingwave")
-else
-  binaries=("meta-node" "compute-node" "frontend" "compactor")
-fi
+binaries=("risingwave")
 
 set -ex
 echo -n "${binaries[*]}" | parallel -d ' ' \
@@ -356,7 +323,6 @@ echo -n "${binaries[*]}" | parallel -d ' ' \
 private = true
 category = "RiseDev - Build"
 description = "Link all binaries to .bin"
-condition = { env_set = ["ENABLE_ALL_IN_ONE"] }
 script = '''
 #!/usr/bin/env bash
 set -e
@@ -375,7 +341,6 @@ category = "RiseDev - Build"
 description = "Copy RisngWave binaries to bin"
 condition = { env_set = ["ENABLE_BUILD_RUST"] }
 dependencies = [
-  "link-standalone-binaries",
   "link-all-in-one-binaries",
   "link-user-bin",
   "codesign-binaries",
@@ -627,6 +592,25 @@ if [ ! -f "${RC_ENV_FILE}" ]; then
 fi
 '''
 
+[tasks.psql-env]
+category = "RiseDev - Start/Stop"
+description = "Dump env configuration for psql"
+dependencies = ["check-risedev-env-file"]
+env_files = ["${PREFIX_CONFIG}/risedev-env"]
+script = '''
+#!/usr/bin/env bash
+cat <<EOF > "${PREFIX_CONFIG}/psql-env"
+export PGHOST=$RW_FRONTEND_LISTEN_ADDRESS
+export PGPORT=$RW_FRONTEND_PORT
+export PGUSER=root
+export PGDATABASE=dev
+EOF
+
+echo "psql environment file is created at ${PREFIX_CONFIG}/psql-env"
+echo "You can source this file to use psql with default connection parameters."
+echo "   $(tput setaf 4)source ${PREFIX_CONFIG}/psql-env$(tput sgr0)"
+'''
+
 [tasks.psql]
 category = "RiseDev - Start/Stop"
 description = "Run local psql client with default connection parameters. You can pass extra arguments to psql."

backwards-compat-tests/scripts/run_local.sh

@@ -37,9 +37,6 @@ ENABLE_KAFKA=true
 # Fetch risingwave binary from release.
 ENABLE_BUILD_RUST=true
 
-# Ensure it will link the all-in-one binary from our release.
-ENABLE_ALL_IN_ONE=true
-
 # ENABLE_RELEASE_PROFILE=true
 EOF
 }

backwards-compat-tests/scripts/utils.sh

@@ -49,6 +49,23 @@ kill_zookeeper() {
   wait_zookeeper_exit
 }
 
+wait_for_process() {
+    process_name="$1"
+
+    while pgrep -x "$process_name" > /dev/null; do
+        echo "Process $process_name is still running... Wait for 1 sec"
+        sleep 1
+    done
+}
+
+wait_all_process_exit() {
+  wait_for_process meta-node
+  wait_for_process compute-node
+  wait_for_process frontend
+  wait_for_process compactor
+  echo "All processes has exited."
+}
+
 # Older versions of RW may not gracefully kill kafka.
 # So we duplicate the definition here.
 kill_cluster() {
@@ -75,6 +92,7 @@ kill_cluster() {
 
   tmux kill-session -t risedev
   test $? -eq 0 || { echo "Failed to stop all RiseDev components."; exit 1; }
+  wait_all_process_exit
 }
 
 run_sql () {

ci/risedev-components.ci.benchmark.env

@@ -1,7 +1,6 @@
 RISEDEV_CONFIGURED=true
 ENABLE_MINIO=true
 ENABLE_ETCD=true
-ENABLE_ALL_IN_ONE=true
 ENABLE_KAFKA=true
 ENABLE_BUILD_RUST=true
 ENABLE_RELEASE_PROFILE=true

ci/risedev-components.ci.env

@@ -1,4 +1,3 @@
 RISEDEV_CONFIGURED=true
 ENABLE_MINIO=true
 ENABLE_ETCD=true
-ENABLE_ALL_IN_ONE=true

ci/risedev-components.ci.source.env

@@ -1,6 +1,5 @@
 RISEDEV_CONFIGURED=true
 ENABLE_MINIO=true
 ENABLE_ETCD=true
-ENABLE_ALL_IN_ONE=true
 ENABLE_KAFKA=true
 ENABLE_PUBSUB=true

ci/scripts/backwards-compat-test.sh

@@ -61,9 +61,6 @@ ENABLE_KAFKA=true
 # Fetch risingwave binary from release.
 ENABLE_BUILD_RUST=false
 
-# Ensure it will link the all-in-one binary from our release.
-ENABLE_ALL_IN_ONE=true
-
 # Use target/debug for simplicity.
 ENABLE_RELEASE_PROFILE=false
 EOF
@@ -94,9 +91,6 @@ ENABLE_KAFKA=true
 # Make sure that it builds
 ENABLE_BUILD_RUST=true
 
-# Ensure it will link the all-in-one binary from our release.
-ENABLE_ALL_IN_ONE=true
-
 # Use target/debug for simplicity.
 ENABLE_RELEASE_PROFILE=false
 EOF

ci/scripts/check.sh

@@ -28,7 +28,7 @@ sccache --show-stats
 sccache --zero-stats
 
 echo "--- Run clippy check (release)"
-cargo clippy --release --all-targets --features "rw-static-link" --locked -- -D warnings
+cargo clippy --release --all-targets --features "rw-static-link,enable_test_epoch_in_release" --locked -- -D warnings
 
 echo "--- Show sccache stats"
 sccache --show-stats

ci/scripts/gen-flamegraph.sh

@@ -159,7 +159,6 @@ ENABLE_KAFKA=true
 ENABLE_COMPUTE_TRACING=true
 ENABLE_BUILD_RUST=true
 ENABLE_RELEASE_PROFILE=true
-ENABLE_ALL_IN_ONE=true
 EOF
 popd
 }

ci/scripts/notify.py

@@ -82,11 +82,11 @@
     "redis-sink-json": ["xinhao"],
     "big-query-sink-json": ["xinhao"],
     "vector-json": ["tao"],
-    "doris-sink": ["xinhao"],
-    "starrocks-sink": ["xinhao"],
-    "deltalake-sink": ["xinhao"],
-    "pinot-sink": ["yiming"],
-    "client-library": ["tao"],
+    "doris-sink-json": ["xinhao"],
+    "starrocks-sink-json": ["xinhao"],
+    "deltalake-sink-json": ["xinhao"],
+    "pinot-sink-json": ["yiming"],
+    "client-library-none": ["tao"],
 }
 
 def get_failed_tests(get_test_status, test_map):

ci/scripts/run-e2e-test.sh

@@ -26,13 +26,24 @@ if [[ $mode == "standalone" ]]; then
   source ci/scripts/standalone-utils.sh
 fi
 
+if [[ $mode == "single-node" ]]; then
+  source ci/scripts/single-node-utils.sh
+fi
+
 cluster_start() {
   if [[ $mode == "standalone" ]]; then
     mkdir -p "$PREFIX_LOG"
     cargo make clean-data
     cargo make pre-start-dev
     start_standalone "$PREFIX_LOG"/standalone.log &
     cargo make dev standalone-minio-etcd
+  elif [[ $mode == "single-node" ]]; then
+    mkdir -p "$PREFIX_LOG"
+    cargo make clean-data
+    cargo make pre-start-dev
+    start_single_node "$PREFIX_LOG"/single-node.log &
+    # Give it a while to make sure the single-node is ready.
+    sleep 3
   else
     cargo make ci-start "$mode"
   fi
@@ -44,6 +55,9 @@ cluster_stop() {
     stop_standalone
     # Don't check standalone logs, they will exceed the limit.
     cargo make kill
+  elif [[ $mode == "single-node" ]]
+  then
+    stop_single_node
   else
     cargo make ci-kill
   fi

ci/scripts/single-node-utils.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export RW_PREFIX=$PWD/.risingwave
+export PREFIX_BIN=./target/debug
+export PREFIX_LOG=$RW_PREFIX/log
+
+# You can fill up this section by consulting
+# .risingwave/log/risedev.log, after calling ./risedev d full.
+# It is expected that minio, etcd will be started after this is called.
+start_single_node() {
+  mkdir -p "$HOME/.risingwave/state_store"
+  mkdir -p "$HOME/.risingwave/meta_store"
+  RUST_BACKTRACE=1 "$PREFIX_BIN"/risingwave >"$1" 2>&1
+}
+
+stop_single_node() {
+  pkill risingwave
+  rm -rf "$HOME/.risingwave/state_store"
+  rm -rf "$HOME/.risingwave/meta_store"
+}
+
+wait_single_node() {
+  set +e
+  timeout 20s bash -c '
+    while true; do
+      echo "Polling every 1s for single_node to be ready for 20s"
+      if psql -h localhost -p 4566 -d dev -U root -c "SELECT 1;" </dev/null
+      then exit 0;
+      else sleep 1;
+      fi
+    done
+  '
+  STATUS=$?
+  set -e
+  if [[ $STATUS -ne 0 ]]; then
+    echo "Single node failed to start with status: $STATUS"
+    exit 1
+  else
+    echo "Single node is ready"
+  fi
+}
+
+restart_single_node() {
+  stop_single_node
+  sleep 5
+  start_single_node "$PREFIX_LOG"/single-node-restarted.log &
+  wait_single_node
+}

ci/workflows/main-cron.yml

@@ -777,6 +777,26 @@ steps:
     timeout_in_minutes: 25
     retry: *auto-retry
 
+  - label: "e2e single-node binary test"
+    key: "e2e-single-node-binary-tests"
+    command: "ci/scripts/e2e-test.sh -p ci-release -m single-node"
+    if: |
+      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-e2e-single-node-tests"
+      || build.env("CI_STEPS") =~ /(^|,)e2e-single-node-tests?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+      - "docslt"
+    plugins:
+      - docker-compose#v4.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 25
+    retry: *auto-retry
+
   - label: "end-to-end test for opendal (parallel)"
     key: "e2e-test-opendal-parallel"
     command: "ci/scripts/e2e-test-parallel-for-opendal.sh -p ci-release"